tier/cli : throw a warning when user issues a detach-tier commit/force

command Backport of http://review.gluster.org/#/c/12297/ > Change-Id: Idf7664d509156ce46ef4308ffc07fb556a0aedd2 > BUG: 1268755 > Signed-off-by: Manikandan Selvaganesh <mselvaga@redhat.com> > Reviewed-on: http://review.gluster.org/12297 > Tested-by: NetBSD Build System <jenkins@build.gluster.org> >Tested-by: Gluster Build System <jenkins@build.gluster.com> > Reviewed-by: mohammed rafi kc <rkavunga@redhat.com> > Reviewed-by: Dan Lambright <dlambrig@redhat.com> Change-Id: Idf7664d509156ce46ef4308ffc07fb556a0aedd2 BUG: 1269035 Signed-off-by: Manikandan Selvaganesh <mselvaga@redhat.com> Reviewed-on: http://review.gluster.org/12305 Tested-by: NetBSD Build System <jenkins@build.gluster.org> Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Dan Lambright <dlambrig@redhat.com> Tested-by: Dan Lambright <dlambrig@redhat.com>
author: Manikandan Selvaganesh <mselvaga@redhat.com> 2015-10-05 12:59:18 +0530
committer: Dan Lambright <dlambrig@redhat.com> 2015-10-08 04:47:42 -0700
commit: 37a553575ff8df7ede86e3c69fb48d77bfaccafe (patch)
tree: 2adfe3304b58d76fe70a07aa64a1df6fc7d8d4c6 /cli/src/cli.h
parent: 4cf1e84ec80fc58f6f978165a81fd8acbcc648f8 (diff)
1 files changed, 1 insertions, 1 deletions
diff --git a/cli/src/cli.h b/cli/src/cli.h
index ccf55761870..59315797eef 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -274,7 +274,7 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
 
 int32_t
 cli_cmd_volume_detach_tier_parse (const char **words, int wordcount,
-                                  dict_t **options);
+                                  dict_t **options, int *question);
 
 int32_t
 cli_cmd_volume_tier_parse (const char **words, int wordcount,
diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index b1643d26c..f60fa85ce 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system
+SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system \
+          playground
 
 CLEANFILES =
diff --git a/xlators/bindings/Makefile.am b/xlators/bindings/Makefile.am
deleted file mode 100644
index f77665802..000000000
--- a/xlators/bindings/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = $(BINDINGS_SUBDIRS)
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
deleted file mode 100644
index c0b9141c6..000000000
--- a/xlators/bindings/python/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-
-xlator_PROGRAMS = python.so
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/bindings
-
-python_PYTHON = gluster.py glustertypes.py glusterstack.py
-
-pythondir = $(xlatordir)/python
-
-python_so_SOURCES = python.c
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
-	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
-
-AM_LDFLAGS = $(PYTHON_LDFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
deleted file mode 100644
index ee0eb1310..000000000
--- a/xlators/bindings/python/src/gluster.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-from glusterstack import *
-import sys
-import inspect
-
-libglusterfs = CDLL("libglusterfs.so")
-_gf_log = libglusterfs._gf_log
-_gf_log.restype = c_int32
-_gf_log.argtypes = [c_char_p, c_char_p, c_char_p, c_int32, c_int, c_char_p]
-
-gf_log_loglevel = c_int.in_dll(libglusterfs, "gf_log_loglevel")
-
-GF_LOG_NONE = 0
-GF_LOG_CRITICAL = 1
-GF_LOG_ERROR = 2
-GF_LOG_WARNING = 3
-GF_LOG_DEBUG = 4
-
-def gf_log(module, level, fmt, *params):
-    if level <= gf_log_loglevel:
-        frame = sys._getframe(1)
-        _gf_log(module, frame.f_code.co_filename, frame.f_code.co_name,
-                frame.f_lineno, level, fmt, *params)
-
-class ComplexTranslator(object):
-    def __init__(self, xlator):
-        self.xlator = xlator_t.from_address(xlator)
-
-    def __getattr__(self, item):
-        return getattr(self.xlator, item)
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
deleted file mode 100644
index ba24c8165..000000000
--- a/xlators/bindings/python/src/glusterstack.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-
-libc = CDLL("libc.so.6")
-calloc = libc.calloc
-calloc.argtypes = [c_int, c_int]
-calloc.restype = c_void_p
-
-# TODO: Can these be done in C somehow?
-def stack_wind(frame, rfn, obj, fn, *params):
-    """Frame is a frame object"""
-    _new = cast(calloc(1, sizeof(call_frame_t)), POINTER(call_frame_t))
-    _new[0].root = frame.root
-    _new[0].next = frame.root[0].frames.next
-    _new[0].prev = pointer(frame.root[0].frames)
-    if frame.root[0].frames.next:
-        frame.root[0].frames.next[0].prev = _new
-    frame.root[0].frames.next = _new
-    _new[0].this = obj
-    # TODO: Type checking like tmp_cbk?
-    _new[0].ret = rfn
-    _new[0].parent = pointer(frame)
-    _new[0].cookie = cast(_new, c_void_p)
-    # TODO: Initialize lock
-    #_new.lock.init()
-    frame.ref_count += 1
-    fn(_new, obj, *params)
-
-def stack_unwind(frame, *params):
-    """Frame is a frame object"""
-    fn = frame[0].ret
-    parent = frame[0].parent[0]
-    parent.ref_count -= 1
-
-    op_ret = params[0]
-    op_err = params[1]
-    params = params[2:]
-    fn(parent, call_frame_t.from_address(frame[0].cookie), parent.this,
-            op_ret, op_err, *params)
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
deleted file mode 100644
index e9069d07c..000000000
--- a/xlators/bindings/python/src/glustertypes.py
+++ /dev/null
@@ -1,167 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-import collections
-
-#
-# Forward declaration of some gluster types
-#
-class call_frame_t(Structure):
-    pass
-
-class call_ctx_t(Structure):
-    pass
-
-class call_pool_t(Structure):
-    pass
-
-class xlator_t(Structure):
-    def _getFirstChild(self):
-        return self.children[0].xlator
-    firstChild = property(_getFirstChild)
-
-class xlator_list_t(Structure):
-    pass
-
-class xlator_fops(Structure):
-    pass
-
-class xlator_mops(Structure):
-    pass
-
-class glusterfs_ctx_t(Structure):
-    pass
-
-class list_head(Structure):
-    pass
-
-class dict_t(Structure):
-    pass
-
-class inode_table_t(Structure):
-    pass
-
-class fd_t(Structure):
-    pass
-
-class iovec(Structure):
-    _fields_ = [
-            ("iov_base", c_void_p),
-            ("iov_len", c_size_t),
-            ]
-
-    def __init__(self, s):
-        self.iov_base = cast(c_char_p(s), c_void_p)
-        self.iov_len = len(s)
-
-    def getBytes(self):
-        return string_at(self.iov_base, self.iov_len)
-
-# This is a pthread_spinlock_t
-# TODO: what happens to volatile-ness?
-gf_lock_t = c_int
-
-uid_t = c_uint32
-gid_t = c_uint32
-pid_t = c_int32
-
-off_t = c_int64
-
-#
-# Function pointer types
-#
-ret_fn_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(call_frame_t),
-                              POINTER(xlator_t), c_int32, c_int32)
-
-fini_fn_t = CFUNCTYPE(None, POINTER(xlator_t))
-init_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t))
-event_notify_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t), c_int32, c_void_p)
-
-list_head._fields_ = [
-        ("next", POINTER(list_head)),
-        ("prev", POINTER(list_head)),
-        ]
-
-call_frame_t._fields_ = [
-        ("root", POINTER(call_ctx_t)),
-        ("parent", POINTER(call_frame_t)),
-        ("next", POINTER(call_frame_t)),
-        ("prev", POINTER(call_frame_t)),
-        ("local", c_void_p),
-        ("this", POINTER(xlator_t)),
-        ("ret", ret_fn_t),
-        ("ref_count", c_int32),
-        ("lock", gf_lock_t),
-        ("cookie", c_void_p),
-        ("op", c_int32),
-        ("type", c_int8),
-        ]
-
-call_ctx_t._fields_ = [
-        ("all_frames", list_head),
-        ("trans", c_void_p),
-        ("pool", call_pool_t),
-        ("unique", c_uint64),
-        ("state", c_void_p),
-        ("uid", uid_t),
-        ("gid", gid_t),
-        ("pid", pid_t),
-        ("frames", call_frame_t),
-        ("req_refs", POINTER(dict_t)),
-        ("rsp_refs", POINTER(dict_t)),
-        ]
-
-xlator_t._fields_ = [
-        ("name", c_char_p),
-        ("type", c_char_p),
-        ("next", POINTER(xlator_t)),
-        ("prev", POINTER(xlator_t)),
-        ("parent", POINTER(xlator_t)),
-        ("children", POINTER(xlator_list_t)),
-        ("fops", POINTER(xlator_fops)),
-        ("mops", POINTER(xlator_mops)),
-        ("fini", fini_fn_t),
-        ("init", init_fn_t),
-        ("notify", event_notify_fn_t),
-        ("options", POINTER(dict_t)),
-        ("ctx", POINTER(glusterfs_ctx_t)),
-        ("itable", POINTER(inode_table_t)),
-        ("ready", c_char),
-        ("private", c_void_p),
-        ]
-
-xlator_list_t._fields_ = [
-        ("xlator", POINTER(xlator_t)),
-        ("next", POINTER(xlator_list_t)),
-        ]
-
-fop_functions = collections.defaultdict(lambda: c_void_p)
-fop_function_names = ['lookup', 'forget', 'stat', 'fstat', 'chmod', 'fchmod',
-        'chown', 'fchown', 'truncate', 'ftruncate', 'utimens', 'access',
-        'readlink', 'mknod', 'mkdir', 'unlink', 'rmdir', 'symlink',
-        'rename', 'link', 'create', 'open', 'readv', 'writev', 'flush',
-        'close', 'fsync', 'opendir', 'readdir', 'closedir', 'fsyncdir',
-        'statfs', 'setxattr', 'getxattr', 'removexattr', 'lk', 'writedir',
-        # TODO: Call backs?
-        ]
-
-fop_writev_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(xlator_t),
-                                  POINTER(fd_t), POINTER(iovec), c_int32,
-                                  off_t)
-
-fop_functions['writev'] = fop_writev_t
-xlator_fops._fields_ = [(f, fop_functions[f]) for f in fop_function_names]
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
deleted file mode 100644
index 3310a2115..000000000
--- a/xlators/bindings/python/src/python.c
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
-   Copyright (c) 2007-2010 Chris AtLee <chris@atlee.ca>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <Python.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-typedef struct
-{
-    char        *scriptname;
-    PyObject    *pXlator;
-    PyObject    *pScriptModule;
-    PyObject    *pGlusterModule;
-    PyThreadState *pInterp;
-
-    PyObject    *pFrameType, *pVectorType, *pFdType;
-} python_private_t;
-
-int32_t
-python_writev (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iovec *vector,
-              int32_t count, 
-              off_t offset)
-{
-  python_private_t *priv = (python_private_t *)this->private;
-  gf_log("python", GF_LOG_DEBUG, "In writev");
-  if (PyObject_HasAttrString(priv->pXlator, "writev"))
-  {
-
-      PyObject *retval = PyObject_CallMethod(priv->pXlator, "writev",
-              "O O O i l",
-              PyObject_CallMethod(priv->pFrameType, "from_address", "O&", PyLong_FromVoidPtr, frame),
-              PyObject_CallMethod(priv->pFdType, "from_address", "O&", PyLong_FromVoidPtr, fd),
-              PyObject_CallMethod(priv->pVectorType, "from_address", "O&", PyLong_FromVoidPtr, vector),
-              count,
-              offset);
-      if (PyErr_Occurred())
-      {
-          PyErr_Print();
-      }
-      Py_XDECREF(retval);
-  }
-  else
-  {
-      return default_writev(frame, this, fd, vector, count, offset);
-  }
-  return 0;
-}
-
-struct xlator_fops fops = {
-    .writev       = python_writev
-};
-
-static PyObject *
-AnonModule_FromFile (const char* fname)
-{
-    // Get the builtins
-    PyThreadState* pThread = PyThreadState_Get();
-    PyObject *pBuiltins = pThread->interp->builtins;
-
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        return NULL;
-    }
-
-    // Create a new dictionary for running code in
-    PyObject *pModuleDict = PyDict_New();
-    PyDict_SetItemString(pModuleDict, "__builtins__", pBuiltins);
-    Py_INCREF(pBuiltins);
-
-    // Run the file in the new context
-    FILE* fp = fopen(fname, "r");
-    PyRun_File(fp, fname, Py_file_input, pModuleDict, pModuleDict);
-    fclose(fp);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-
-    // Create an object to hold the new context
-    PyRun_String("class ModuleWrapper(object):\n\tpass\n", Py_single_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-    PyObject *pModule = PyRun_String("ModuleWrapper()", Py_eval_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_XDECREF(pModule);
-        return NULL;
-    }
-
-    // Set the new context's dictionary to the one we used to run the code
-    // inside
-    PyObject_SetAttrString(pModule, "__dict__", pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_DECREF(pModule);
-        return NULL;
-    }
-
-    return pModule;
-}
-
-int32_t
-init (xlator_t *this)
-{
-  // This is ok to call more than once per process
-  Py_InitializeEx(0);
-
-  if (!this->children) {
-    gf_log ("python", GF_LOG_ERROR, 
-            "FATAL: python should have exactly one child");
-    return -1;
-  }
-
-  python_private_t *priv = CALLOC (sizeof (python_private_t), 1);
-  ERR_ABORT (priv);
-
-  data_t *scriptname = dict_get (this->options, "scriptname");
-  if (scriptname) {
-      priv->scriptname = data_to_str(scriptname);
-  } else {
-      gf_log("python", GF_LOG_ERROR,
-              "FATAL: python requires the scriptname parameter");
-      return -1;
-  }
-
-  priv->pInterp = Py_NewInterpreter();
-    
-  // Adjust python's path
-  PyObject *syspath = PySys_GetObject("path");
-  PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
-  PyList_Append(syspath, path);
-  Py_DECREF(path);
-
-  gf_log("python", GF_LOG_DEBUG,
-          "Loading gluster module");
-
-  priv->pGlusterModule = PyImport_ImportModule("gluster");
-  if (PyErr_Occurred())
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  priv->pFrameType = PyObject_GetAttrString(priv->pGlusterModule, "call_frame_t");
-  priv->pFdType = PyObject_GetAttrString(priv->pGlusterModule, "fd_t");
-  priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
-
-  gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
-  
-  priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
-  if (!priv->pScriptModule || PyErr_Occurred())
-  {
-      gf_log("python", GF_LOG_ERROR, "Error loading %s", priv->scriptname);
-      PyErr_Print();
-      return -1;
-  }
-
-  if (!PyObject_HasAttrString(priv->pScriptModule, "xlator"))
-  {
-      gf_log("python", GF_LOG_ERROR, "%s does not have a xlator attribute", priv->scriptname);
-      return -1;
-  }
-  gf_log("python", GF_LOG_DEBUG, "Instantiating translator");
-  priv->pXlator = PyObject_CallMethod(priv->pScriptModule, "xlator", "O&",
-          PyLong_FromVoidPtr, this);
-  if (PyErr_Occurred() || !priv->pXlator)
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  this->private = priv;
-
-  gf_log ("python", GF_LOG_DEBUG, "python xlator loaded");
-  return 0;
-}
-
-void 
-fini (xlator_t *this)
-{
-  python_private_t *priv = (python_private_t*)(this->private);
-  Py_DECREF(priv->pXlator);
-  Py_DECREF(priv->pScriptModule);
-  Py_DECREF(priv->pGlusterModule);
-  Py_DECREF(priv->pFrameType);
-  Py_DECREF(priv->pFdType);
-  Py_DECREF(priv->pVectorType);
-  Py_EndInterpreter(priv->pInterp);
-  return;
-}
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
deleted file mode 100644
index 507455c85..000000000
--- a/xlators/bindings/python/src/testxlator.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-
-"""
-This is a test translator written in python.
-
-Important things to note:
-    This file must be import-able from glusterfsd.  This probably means
-    setting PYTHONPATH to where this file is located.
-
-    This file must have a top-level xlator class object that will be
-    used to instantiate individual translators.
-"""
-from gluster import *
-
-class MyXlator(ComplexTranslator):
-    name = "MyXlator"
-    def writev_cbk(self, frame, cookie, op_ret, op_errno, buf):
-        stack_unwind(frame, op_ret, op_errno, buf)
-        return 0
-
-    def writev(self, frame, fd, vector, count, offset):
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-        # TODO: Use cookie to pass this to writev_cbk
-        old_count = vector.iov_len
-
-        data = vector.getBytes().encode("zlib")
-
-        vector = iovec(data)
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-
-        @ret_fn_t
-        def rfn(frame, prev, this, op_ret, op_errno, *params):
-            if len(params) == 0:
-                params = [0]
-            return self.writev_cbk(frame, prev, old_count, op_errno, *params)
-
-        stack_wind(frame, rfn, self.firstChild,
-                self.firstChild[0].fops[0].writev, fd, vector, count, offset)
-        return 0
-
-xlator = MyXlator
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index ed0901813..ea5a90abb 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,27 +1,38 @@
 xlator_LTLIBRARIES = afr.la pump.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c afr-self-heal-algorithm.c afr-lk-common.c afr-self-heald.c $(top_builddir)/xlators/lib/src/libxlator.c
+afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
+	afr-inode-write.c afr-open.c afr-transaction.c afr-lk-common.c \
+	afr-read-txn.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-afr_la_LDFLAGS = -module -avoidversion
-afr_la_SOURCES = $(afr_common_source) afr.c
+AFR_SELFHEAL_SOURCES = afr-self-heal-common.c afr-self-heal-data.c \
+	afr-self-heal-entry.c afr-self-heal-metadata.c afr-self-heald.c \
+	afr-self-heal-name.c
+
+afr_la_LDFLAGS = -module -avoid-version
+afr_la_SOURCES = $(afr_common_source) $(AFR_SELFHEAL_SOURCES) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-pump_la_LDFLAGS = -module -avoidversion
-pump_la_SOURCES =  $(afr_common_source) pump.c
+pump_la_LDFLAGS = -module -avoid-version
+pump_la_SOURCES =  $(afr_common_source) $(AFR_SELFHEAL_SOURCES) pump.c
 pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c afr-self-heald.h $(top_builddir)/xlators/lib/src/libxlator.h $(top_builddir)/glusterfsd/src/glusterfsd.h
+noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
+	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-mem-types.h \
+	afr-common.c afr-self-heald.h pump.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/contrib/md5 -shared -nostartfiles $(GF_CFLAGS) \
-	    -I$(top_srcdir)/xlators/lib/src -I$(top_srcdir)/rpc/rpc-lib/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/replicate.so
-	rm -f $(DESTDIR)$(xlatordir)/pump.so
 
 install-data-hook:
 	ln -sf afr.so $(DESTDIR)$(xlatordir)/replicate.so
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 311860170..6bd231600 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -54,760 +45,821 @@
 #include "afr-dir-write.h"
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
 #include "afr-self-heald.h"
-#include "pump.h"
 
-#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000200000000ULL
-#define AFR_ICTX_SPLIT_BRAIN_MASK      0x0000000100000000ULL
-#define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
 
-int
-afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
-                                gf_boolean_t fail_conflict);
-void
-afr_children_copy (int32_t *dst, int32_t *src, unsigned int child_count)
+call_frame_t *
+afr_copy_frame (call_frame_t *base)
 {
-        int     i = 0;
+	afr_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	int op_errno = 0;
+
+	frame = copy_frame (base);
+	if (!frame)
+		return NULL;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local) {
+		AFR_STACK_DESTROY (frame);
+		return NULL;
+	}
 
-        for (i = 0; i < child_count; i++)
-                dst[i] = src[i];
+	return frame;
 }
 
-void
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path)
+/*
+ * INODE CTX 64-bit VALUE FORMAT FOR SMALL (<= 16) SUBVOL COUNTS:
+ *
+ * |<----------   64bit   ------------>|
+ *  63           32 31    16 15       0
+ * |   EVENT_GEN   |  DATA  | METADATA |
+ *
+ *
+ *  METADATA (bit-0 .. bit-15): bitmap representing subvolumes from which
+ *                              metadata can be attempted to be read.
+ *
+ *                              bit-0 => priv->subvolumes[0]
+ *                              bit-1 => priv->subvolumes[1]
+ *                              ... etc. till bit-15
+ *
+ *  DATA (bit-16 .. bit-31): bitmap representing subvolumes from which data
+ *                           can be attempted to be read.
+ *
+ *                           bit-16 => priv->subvolumes[0]
+ *                           bit-17 => priv->subvolumes[1]
+ *                           ... etc. till bit-31
+ *
+ *  EVENT_GEN (bit-32 .. bit-63): event generation (i.e priv->event_generation)
+ *                                when DATA and METADATA was last updated.
+ *
+ *                                If EVENT_GEN is < priv->event_generation,
+ *                                or is 0, it means afr_inode_refresh() needs
+ *                                to be called to recalculate the bitmaps.
+ */
+
+int
+__afr_inode_read_subvol_get_small (inode_t *inode, xlator_t *this,
+				   unsigned char *data, unsigned char *metadata,
+				   int *event_p)
 {
-        int             i           = 0;
-        afr_private_t   *priv       = NULL;
-        int             ret         = 0;
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint32_t event = 0;
+	uint64_t val = 0;
+	int i = 0;
 
-        priv   = this->private;
+	priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
-                                       3 * sizeof(int32_t));
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: Unable to set dict value for %s",
-                                path, priv->pending_key[i]);
-                /* 3 = data+metadata+entry */
-        }
+	ret = __inode_ctx_get (inode, this, &val);
+	if (ret < 0)
+		return ret;
+
+	metadatamap = (val & 0x000000000000ffff);
+	datamap =     (val & 0x00000000ffff0000) >> 16;
+	event =       (val & 0xffffffff00000000) >> 32;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (metadata)
+			metadata[i] = (metadatamap >> i) & 1;
+		if (data)
+			data[i] = (datamap >> i) & 1;
+	}
+
+	if (event_p)
+		*event_p = event;
+	return ret;
 }
 
+
 int
-afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
-                              dict_t *xattr_req, loc_t *loc, void **gfid_req)
+__afr_inode_read_subvol_set_small (inode_t *inode, xlator_t *this,
+				   unsigned char *data, unsigned char *metadata,
+				   int event)
 {
-        int     ret = -ENOMEM;
+	afr_private_t *priv = NULL;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint64_t val = 0;
+	int i = 0;
 
-        GF_ASSERT (gfid_req);
+	priv = this->private;
 
-        *gfid_req = NULL;
-        local->xattr_req = dict_new ();
-        if (!local->xattr_req)
-                goto out;
-        if (xattr_req)
-                dict_copy (xattr_req, local->xattr_req);
+	for (i = 0; i < priv->child_count; i++) {
+		if (data[i])
+			datamap |= (1 << i);
+		if (metadata[i])
+			metadatamap |= (1 << i);
+	}
 
-        afr_xattr_req_prepare (this, local->xattr_req, loc->path);
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_INODELK_COUNT);
-        }
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
-        }
+	val = ((uint64_t) metadatamap) |
+		(((uint64_t) datamap) << 16) |
+		(((uint64_t) event) << 32);
 
-        ret = dict_get_ptr (local->xattr_req, "gfid-req", gfid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: failed to get the gfid from dict", loc->path);
-                *gfid_req = NULL;
-        } else {
-                if (loc->parent != NULL)
-                        dict_del (local->xattr_req, "gfid-req");
-        }
-        ret = 0;
-out:
-        return ret;
+	return __inode_ctx_set (inode, this, &val);
 }
 
-void
-afr_lookup_save_gfid (uuid_t dst, void* new, const loc_t *loc)
+
+int
+__afr_inode_read_subvol_reset_small (inode_t *inode, xlator_t *this)
 {
-        inode_t  *inode = NULL;
+	int ret = -1;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint32_t event = 0;
+	uint64_t val = 0;
 
-        inode = loc->inode;
-        if (inode && !uuid_is_null (inode->gfid))
-                uuid_copy (dst, inode->gfid);
-        else if (!uuid_is_null (loc->gfid))
-                uuid_copy (dst, loc->gfid);
-        else if (new && !uuid_is_null (new))
-                uuid_copy (dst, new);
+	ret = __inode_ctx_get (inode, this, &val);
+	(void) ret;
+
+	metadatamap = (val & 0x000000000000ffff) >> 0;
+	datamap =     (val & 0x00000000ffff0000) >> 16;
+	event = 0;
+
+	val = ((uint64_t) metadatamap) |
+		(((uint64_t) datamap) << 16) |
+		(((uint64_t) event) << 32);
+
+	return __inode_ctx_set (inode, this, &val);
 }
 
+
 int
-afr_errno_count (int32_t *children, int *child_errno,
-                 unsigned int child_count, int32_t op_errno)
+__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			     unsigned char *data, unsigned char *metadata,
+			     int *event_p)
 {
-        int i = 0;
-        int errno_count = 0;
-        int child = 0;
+	afr_private_t *priv = NULL;
+	int ret = -1;
 
-        for (i = 0; i < child_count; i++) {
-                if (children) {
-                        child = children[i];
-                        if (child == -1)
-                                break;
-                } else {
-                        child = i;
-                }
-                if (child_errno[child] == op_errno)
-                        errno_count++;
-        }
-        return errno_count;
-}
+	priv = this->private;
 
-int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
-{
-        int ret       = 0;
-        uuid_t *pgfid = NULL;
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_get_small (inode, this, data,
+							 metadata, event_p);
+	else
+		/* TBD: allocate structure with array and read from it */
+		ret = -1;
 
-        GF_ASSERT (gfid);
+	return ret;
+}
 
-        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!pgfid) {
-                ret = -1;
-                goto out;
-        }
 
-        uuid_copy (*pgfid, gfid);
+int
+__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
+			     unsigned char *metadata, int event)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
 
-        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "gfid set failed");
+	priv = this->private;
 
-out:
-        if (ret && pgfid)
-                GF_FREE (pgfid);
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_set_small (inode, this, data,
+							 metadata, event);
+	else
+		ret = -1;
 
-        return ret;
+	return ret;
 }
 
-afr_inode_ctx_t*
-afr_inode_ctx_get_from_addr (uint64_t addr, int32_t child_count)
+
+int
+__afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
 {
-        int             ret  = -1;
-        afr_inode_ctx_t *ctx = NULL;
-        size_t          size = 0;
+	afr_private_t *priv = NULL;
+	int ret = -1;
 
-        GF_ASSERT (child_count > 0);
+	priv = this->private;
 
-        if (!addr) {
-                ctx = GF_CALLOC (1, sizeof (*ctx),
-                                 gf_afr_mt_inode_ctx_t);
-                if (!ctx)
-                        goto out;
-                size = sizeof (*ctx->fresh_children);
-                ctx->fresh_children = GF_CALLOC (child_count, size,
-                                                 gf_afr_mt_int32_t);
-                if (!ctx->fresh_children)
-                        goto out;
-        } else {
-                ctx = (afr_inode_ctx_t*) (long) addr;
-        }
-        ret = 0;
-out:
-        if (ret && ctx) {
-                if (ctx->fresh_children)
-                        GF_FREE (ctx->fresh_children);
-                GF_FREE (ctx);
-                ctx = NULL;
-        }
-        return ctx;
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_reset_small (inode, this);
+	else
+		ret = -1;
+
+	return ret;
 }
 
-void
-afr_inode_get_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
+
+int
+afr_inode_read_subvol_get (inode_t *inode, xlator_t *this, unsigned char *data,
+			   unsigned char *metadata, int *event_p)
 {
-        GF_ASSERT (inode);
-        GF_ASSERT (params);
+	int ret = -1;
 
-        int             ret = 0;
-        afr_inode_ctx_t *ctx = NULL;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
-        uint64_t        ctx_addr   = 0;
-        int32_t         read_child = -1;
-        int32_t         *fresh_children = NULL;
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_get (inode, this, data,
+						   metadata, event_p);
+	}
+	UNLOCK(&inode->lock);
 
-        priv = this->private;
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        goto unlock;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
-                if (!ctx)
-                        goto unlock;
-                switch (params->op) {
-                case AFR_INODE_GET_READ_CTX:
-                        fresh_children = params->u.read_ctx.children;
-                        read_child = (int32_t)(ctx->masks &
-                                               AFR_ICTX_READ_CHILD_MASK);
-                        params->u.read_ctx.read_child = read_child;
-                        if (!fresh_children)
-                                goto unlock;
-                        for (i = 0; i < priv->child_count; i++)
-                                fresh_children[i] = ctx->fresh_children[i];
-                        break;
-                case AFR_INODE_GET_OPENDIR_DONE:
-                        params->u.value = _gf_false;
-                        if (ctx->masks & AFR_ICTX_OPENDIR_DONE_MASK)
-                                params->u.value = _gf_true;
-                        break;
-                case AFR_INODE_GET_SPLIT_BRAIN:
-                        params->u.value = _gf_false;
-                        if (ctx->masks & AFR_ICTX_SPLIT_BRAIN_MASK)
-                                params->u.value = _gf_true;
-                        ;
-                        break;
-                default:
-                        GF_ASSERT (0);
-                        break;
-                }
-        }
-unlock:
-        UNLOCK (&inode->lock);
+	return ret;
 }
 
-gf_boolean_t
-afr_is_split_brain (xlator_t *this, inode_t *inode)
+
+int
+afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
+			   unsigned char *metadata, int event)
 {
-        afr_inode_params_t params = {0};
+	int ret = -1;
 
-        params.op = AFR_INODE_GET_SPLIT_BRAIN;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.value;
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_set (inode, this, data, metadata,
+						   event);
+	}
+	UNLOCK(&inode->lock);
+
+	return ret;
 }
 
-gf_boolean_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode)
+
+int
+afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
 {
-        afr_inode_params_t params = {0};
+	int ret = -1;
+
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_reset (inode, this);
+	}
+	UNLOCK(&inode->lock);
 
-        params.op = AFR_INODE_GET_OPENDIR_DONE;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.value;
+	return ret;
 }
 
 
-int32_t
-afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children)
+int
+afr_accused_fill (xlator_t *this, dict_t *xdata, unsigned char *accused,
+		  afr_transaction_type type)
 {
-        afr_inode_params_t      params = {0};
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int idx = afr_index_for_transaction_type (type);
+	void *pending_raw = NULL;
+	int pending[3];
+	int ret = 0;
 
-        params.op = AFR_INODE_GET_READ_CTX;
-        params.u.read_ctx.children = fresh_children;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.read_ctx.read_child;
-}
+	priv = this->private;
 
-void
-afr_inode_ctx_set_read_child (afr_inode_ctx_t *ctx, int32_t read_child)
-{
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask         = 0;
+	for (i = 0; i < priv->child_count; i++) {
+		ret = dict_get_ptr (xdata, priv->pending_key[i],
+				    &pending_raw);
+		if (ret) /* no pending flags */
+			continue;
+		memcpy (pending, pending_raw, sizeof(pending));
 
-        remaining_mask = (~AFR_ICTX_READ_CHILD_MASK & ctx->masks);
-        mask = (AFR_ICTX_READ_CHILD_MASK & read_child);
-        ctx->masks = remaining_mask | mask;
+		if (ntoh32 (pending[idx]))
+			accused[i] = 1;
+	}
+
+	return 0;
 }
 
-void
-afr_inode_ctx_set_read_ctx (afr_inode_ctx_t *ctx, int32_t read_child,
-                            int32_t *fresh_children, int32_t child_count)
+
+int
+afr_accuse_smallfiles (xlator_t *this, struct afr_reply *replies,
+		       unsigned char *data_accused)
 {
-        int             i            = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	uint64_t maxsize = 0;
 
-        afr_inode_ctx_set_read_child (ctx, read_child);
-        for (i = 0; i < child_count; i++) {
-                if (fresh_children)
-                        ctx->fresh_children[i] = fresh_children[i];
-                else
-                        ctx->fresh_children[i] = -1;
-        }
-}
+	priv = this->private;
 
-void
-afr_inode_ctx_rm_stale_children (afr_inode_ctx_t *ctx, int32_t *stale_children,
-                                 int32_t child_count)
-{
-        int             i            = 0;
-        int32_t         read_child   = -1;
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i])
+			continue;
+		if (replies[i].poststat.ia_size > maxsize)
+			maxsize = replies[i].poststat.ia_size;
+	}
 
-        GF_ASSERT (stale_children);
-        for (i = 0; i < child_count; i++) {
-                if (stale_children[i] == -1)
-                        break;
-                afr_children_rm_child (ctx->fresh_children,
-                                       stale_children[i], child_count);
-        }
-        read_child = (int32_t)(ctx->masks & AFR_ICTX_READ_CHILD_MASK);
-        if (!afr_is_child_present (ctx->fresh_children, child_count,
-                                   read_child))
-                afr_inode_ctx_set_read_child (ctx, ctx->fresh_children[0]);
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i])
+			continue;
+		if (replies[i].poststat.ia_size < maxsize)
+			data_accused[i] = 1;
+	}
+
+	return 0;
 }
 
-void
-afr_inode_ctx_set_opendir_done (afr_inode_ctx_t *ctx)
-{
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask = 0;
 
-        remaining_mask = (~AFR_ICTX_OPENDIR_DONE_MASK & ctx->masks);
-        mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_OPENDIR_DONE_MASK);
-        ctx->masks = remaining_mask | mask;
+int
+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	struct afr_reply *replies = NULL;
+	int event_generation = 0;
+	int i = 0;
+	unsigned char *data_accused = NULL;
+	unsigned char *metadata_accused = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	int ret = 0;
+
+	local = frame->local;
+	priv = this->private;
+	replies = local->replies;
+	event_generation = local->event_generation;
+
+	data_accused = alloca0 (priv->child_count);
+	data_readable = alloca0 (priv->child_count);
+	metadata_accused = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+
+	for (i = 0; i < priv->child_count; i++) {
+		data_readable[i] = 1;
+		metadata_readable[i] = 1;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid) {
+			data_readable[i] = 0;
+			metadata_readable[i] = 0;
+			continue;
+		}
+
+		if (replies[i].op_ret == -1) {
+			data_readable[i] = 0;
+			metadata_readable[i] = 0;
+			continue;
+		}
+
+		afr_accused_fill (this, replies[i].xdata, data_accused,
+				  (inode->ia_type == IA_IFDIR) ?
+				   AFR_ENTRY_TRANSACTION : AFR_DATA_TRANSACTION);
+
+		afr_accused_fill (this, replies[i].xdata,
+				  metadata_accused, AFR_METADATA_TRANSACTION);
+
+	}
+
+	if (inode->ia_type != IA_IFDIR)
+		afr_accuse_smallfiles (this, replies, data_accused);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i]) {
+			data_readable[i] = 0;
+			ret = 1;
+		}
+		if (metadata_accused[i]) {
+			metadata_readable[i] = 0;
+			ret = 1;
+		}
+	}
+
+	afr_inode_read_subvol_set (inode, this, data_readable,
+				   metadata_readable, event_generation);
+	return ret;
 }
 
-void
-afr_inode_ctx_set_splitbrain (afr_inode_ctx_t *ctx, gf_boolean_t set)
-{
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask = 0;
 
-        if (set) {
-                remaining_mask = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-                mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_SPLIT_BRAIN_MASK);
-                ctx->masks = remaining_mask | mask;
-        } else {
-                ctx->masks = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-        }
+
+int
+afr_refresh_selfheal_done (int ret, call_frame_t *heal, void *opaque)
+{
+	if (heal)
+		STACK_DESTROY (heal->root);
+	return 0;
 }
 
-void
-afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
+int
+afr_inode_refresh_err (call_frame_t *frame, xlator_t *this)
 {
-        GF_ASSERT (inode);
-        GF_ASSERT (params);
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int err = 0;
 
-        int             ret = 0;
-        afr_inode_ctx_t *ctx            = NULL;
-        afr_private_t   *priv           = NULL;
-        uint64_t        ctx_addr        = 0;
-        gf_boolean_t    set             = _gf_false;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children = NULL;
-        int32_t         *stale_children = NULL;
+	local = frame->local;
+	priv = this->private;
 
-        priv = this->private;
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        ctx_addr = 0;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
-                if (!ctx)
-                        goto unlock;
-                switch (params->op) {
-                case AFR_INODE_SET_READ_CTX:
-                        read_child = params->u.read_ctx.read_child;
-                        fresh_children = params->u.read_ctx.children;
-                        afr_inode_ctx_set_read_ctx (ctx, read_child,
-                                                    fresh_children,
-                                                    priv->child_count);
-                        break;
-                case AFR_INODE_RM_STALE_CHILDREN:
-                        stale_children = params->u.read_ctx.children;
-                        afr_inode_ctx_rm_stale_children (ctx,
-                                                         stale_children,
-                                                         priv->child_count);
-                        break;
-                case AFR_INODE_SET_OPENDIR_DONE:
-                        afr_inode_ctx_set_opendir_done (ctx);
-                        break;
-                case AFR_INODE_SET_SPLIT_BRAIN:
-                        set = params->u.value;
-                        afr_inode_ctx_set_splitbrain (ctx, set);
-                        break;
-                default:
-                        GF_ASSERT (0);
-                        break;
-                }
-                ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
-                if (ret) {
-                        gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
-                                          "set the inode ctx (%s)",
-                                          uuid_utoa (inode->gfid));
-                }
-        }
-unlock:
-        UNLOCK (&inode->lock);
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].valid && !local->replies[i].op_ret) {
+			err = 0;
+			goto ret;
+		}
+	}
+
+	err = afr_final_errno (local, priv);
+ret:
+	return -err;
 }
 
-void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set)
+
+int
+afr_refresh_selfheal_wrap (void *opaque)
 {
-        afr_inode_params_t      params = {0};
+	call_frame_t *frame = opaque;
+	afr_local_t *local = NULL;
+	xlator_t *this = NULL;
+	int err = 0;
 
-        params.op = AFR_INODE_SET_SPLIT_BRAIN;
-        params.u.value          = set;
-        afr_inode_set_ctx (this, inode, &params);
-}
+	local = frame->local;
+	this = frame->this;
 
-void
-afr_set_opendir_done (xlator_t *this, inode_t *inode)
-{
-        afr_inode_params_t params = {0};
+	afr_selfheal (frame->this, local->refreshinode->gfid);
+
+	afr_selfheal_unlocked_discover (frame, local->refreshinode,
+					local->refreshinode->gfid,
+					local->replies);
+
+	afr_replies_interpret (frame, this, local->refreshinode);
+
+	err = afr_inode_refresh_err (frame, this);
+
+	afr_replies_wipe (local, this->private);
 
-        params.op = AFR_INODE_SET_OPENDIR_DONE;
-        afr_inode_set_ctx (this, inode, &params);
+	local->refreshfn (frame, this, err);
+
+	return 0;
 }
 
-void
-afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
-                        int32_t *fresh_children)
+
+gf_boolean_t
+afr_selfheal_enabled (xlator_t *this)
 {
-        afr_inode_params_t params = {0};
-        afr_private_t      *priv  = NULL;
+	afr_private_t *priv = NULL;
+	gf_boolean_t data = _gf_false;
 
-        priv = this->private;
-        GF_ASSERT (read_child >= 0);
-        GF_ASSERT (fresh_children);
-        GF_ASSERT (afr_is_child_present (fresh_children, priv->child_count,
-                                         read_child));
+	priv = this->private;
 
-        params.op = AFR_INODE_SET_READ_CTX;
-        params.u.read_ctx.read_child     = read_child;
-        params.u.read_ctx.children = fresh_children;
-        afr_inode_set_ctx (this, inode, &params);
+	gf_string2boolean (priv->data_self_heal, &data);
+
+	return data || priv->metadata_self_heal || priv->entry_self_heal;
 }
 
-void
-afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
-                             int32_t *stale_children)
+
+
+int
+afr_inode_refresh_done (call_frame_t *frame, xlator_t *this)
 {
-        afr_inode_params_t params = {0};
+	call_frame_t *heal = NULL;
+	afr_local_t *local = NULL;
+	int ret = 0;
+	int err = 0;
+
+	local = frame->local;
+
+	ret = afr_replies_interpret (frame, this, local->refreshinode);
+
+	err = afr_inode_refresh_err (frame, this);
+
+	afr_replies_wipe (local, this->private);
 
-        GF_ASSERT (stale_children);
+	if (ret && afr_selfheal_enabled (this)) {
+		heal = copy_frame (frame);
+		if (heal)
+			heal->root->pid = -1;
+		ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,
+				    afr_refresh_selfheal_done, heal, frame);
+		if (ret)
+			goto refresh_done;
+	} else {
+	refresh_done:
+		local->refreshfn (frame, this, err);
+	}
 
-        params.op = AFR_INODE_RM_STALE_CHILDREN;
-        params.u.read_ctx.children = stale_children;
-        afr_inode_set_ctx (this, inode, &params);
+	return 0;
 }
 
-gf_boolean_t
-afr_is_source_child (int32_t *sources, int32_t child_count, int32_t child)
+
+int
+afr_inode_refresh_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			      int op_ret, int op_errno, inode_t *inode,
+			      struct iatt *buf, dict_t *xdata, struct iatt *par)
 {
-        gf_boolean_t             source_xattrs = _gf_false;
+	afr_local_t *local = NULL;
+	int call_child = (long) cookie;
+	int call_count = 0;
 
-        GF_ASSERT (child < child_count);
+	local = frame->local;
 
-        if ((child >= 0) && (child < child_count) &&
-             sources[child]) {
-                source_xattrs = _gf_true;
-        }
-        return source_xattrs;
+	local->replies[call_child].valid = 1;
+	local->replies[call_child].op_ret = op_ret;
+	local->replies[call_child].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[call_child].poststat = *buf;
+		local->replies[call_child].postparent = *par;
+		local->replies[call_child].xdata = dict_ref (xdata);
+	}
+
+	call_count = afr_frame_return (frame);
+
+	if (call_count == 0)
+		afr_inode_refresh_done (frame, this);
+
+	return 0;
 }
 
-gf_boolean_t
-afr_is_child_present (int32_t *success_children, int32_t child_count,
-                      int32_t child)
+
+int
+afr_inode_refresh_subvol (call_frame_t *frame, xlator_t *this, int i,
+			  inode_t *inode, dict_t *xdata)
 {
-        gf_boolean_t             success_child = _gf_false;
-        int                      i = 0;
+	loc_t loc = {0, };
+	afr_private_t *priv = NULL;
 
-        GF_ASSERT (child < child_count);
+	priv = this->private;
 
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                if (child == success_children[i]) {
-                        success_child = _gf_true;
-                        break;
-                }
-        }
-        return success_child;
+	loc.inode = inode;
+	uuid_copy (loc.gfid, inode->gfid);
+
+	STACK_WIND_COOKIE (frame, afr_inode_refresh_subvol_cbk,
+			   (void *) (long) i, priv->children[i],
+			   priv->children[i]->fops->lookup, &loc, xdata);
+	return 0;
 }
 
-gf_boolean_t
-afr_is_read_child (int32_t *success_children, int32_t *sources,
-                   int32_t child_count, int32_t child)
+
+int
+afr_inode_refresh_do (call_frame_t *frame, xlator_t *this)
 {
-        gf_boolean_t             success_child = _gf_false;
-        gf_boolean_t             source        = _gf_false;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
+	int i = 0;
+	dict_t *xdata = NULL;
 
-        GF_ASSERT (success_children);
-        GF_ASSERT (child_count > 0);
+	priv = this->private;
+	local = frame->local;
 
-        success_child = afr_is_child_present (success_children, child_count,
-                                              child);
-        if (!success_child)
-                goto out;
-        if (NULL == sources) {
-                source = _gf_true;
-                goto out;
-        }
-        source = afr_is_source_child (sources, child_count, child);
-out:
-        return (success_child && source);
+	afr_replies_wipe (local, priv);
+
+	xdata = dict_new ();
+	if (!xdata) {
+		afr_inode_refresh_done (frame, this);
+		return 0;
+	}
+
+	if (afr_xattr_req_prepare (this, xdata) != 0) {
+		dict_unref (xdata);
+		afr_inode_refresh_done (frame, this);
+		return 0;
+	}
+
+	local->call_count = AFR_COUNT (local->child_up, priv->child_count);
+
+	call_count = local->call_count;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->child_up[i])
+			continue;
+
+		afr_inode_refresh_subvol (frame, this, i, local->refreshinode,
+					  xdata);
+
+		if (!--call_count)
+			break;
+	}
+
+	dict_unref (xdata);
+
+	return 0;
 }
 
-/* If sources is NULL the xattrs are assumed to be of source for all
- * success_children.
- */
+
 int
-afr_select_read_child_from_policy (int32_t *success_children, int32_t child_count,
-                                   int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources)
+afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		   afr_inode_refresh_cbk_t refreshfn)
 {
-        int32_t                  read_child   = -1;
-        int                      i            = 0;
+	afr_local_t *local = NULL;
 
-        GF_ASSERT (success_children);
+	local = frame->local;
 
-        read_child = prev_read_child;
-        if (afr_is_read_child (success_children, sources, child_count,
-                               read_child))
-                goto out;
+	local->refreshfn = refreshfn;
 
-        read_child = config_read_child;
-        if (afr_is_read_child (success_children, sources, child_count,
-                               read_child))
-                goto out;
+	if (local->refreshinode) {
+		inode_unref (local->refreshinode);
+		local->refreshinode = NULL;
+	}
 
-        for (i = 0; i < child_count; i++) {
-                read_child = success_children[i];
-                if (read_child < 0)
-                        break;
-                if (afr_is_read_child (success_children, sources, child_count,
-                                       read_child))
-                        goto out;
-        }
-        read_child = -1;
+	local->refreshinode = inode_ref (inode);
 
-out:
-        return read_child;
+	afr_inode_refresh_do (frame, this);
+
+	return 0;
 }
 
-/* This function should be used when all the success_children are sources
- */
-void
-afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
-                              int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child)
-{
-        int                      read_child = -1;
-        afr_private_t            *priv = NULL;
 
-        priv = this->private;
-        read_child = afr_select_read_child_from_policy (fresh_children,
-                                                        priv->child_count,
-                                                        prev_read_child,
-                                                        config_read_child,
-                                                        NULL);
-        if (read_child >= 0)
-                afr_inode_set_read_ctx (this, inode, read_child,
-                                        fresh_children);
-}
-
-/* afr_next_call_child ()
- * This is a common function used by all the read-type fops
- * This function should not be called with the inode's read_children array.
- * The fop's handler should make a copy of the inode's read_children,
- * preferred read_child into the local vars, because while this function is
- * in execution there is a chance for inode's read_ctx to change.
- */
-int32_t
-afr_next_call_child (int32_t *fresh_children, unsigned char *child_up,
-                     size_t child_count, int32_t *last_index,
-                     int32_t read_child)
+int
+afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req)
 {
-        int             next_index      = 0;
-        int32_t         next_call_child = -1;
+        int             i           = 0;
+        afr_private_t   *priv       = NULL;
+        int             ret         = 0;
 
-        GF_ASSERT (last_index);
+        priv   = this->private;
 
-        next_index = *last_index;
-retry:
-        next_index++;
-        if ((next_index >= child_count) ||
-           (fresh_children[next_index] == -1))
-                goto out;
-        if ((fresh_children[next_index] == read_child) ||
-           (!child_up[fresh_children[next_index]]))
-                goto retry;
-        *last_index = next_index;
-        next_call_child = fresh_children[next_index];
-out:
-        return next_call_child;
+        for (i = 0; i < priv->child_count; i++) {
+                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
+                                       AFR_NUM_CHANGE_LOGS * sizeof(int));
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Unable to set dict value for %s",
+                                priv->pending_key[i]);
+                /* 3 = data+metadata+entry */
+        }
+        ret = dict_set_uint64 (xattr_req, AFR_DIRTY,
+			       AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "failed to set dirty "
+                        "query flag");
+        }
+
+	return ret;
 }
 
- /* This function should not be called with the inode's read_children array.
- * The fop's handler should make a copy of the inode's read_children,
- * preferred read_child into the local vars, because while this function is
- * in execution there is a chance for inode's read_ctx to change.
- */
-int32_t
-afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
-                    int32_t *fresh_children,
-                    int32_t *call_child, int32_t *last_index)
+int
+afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
+                              dict_t *xattr_req, loc_t *loc)
 {
-        int             ret   = 0;
-        afr_private_t   *priv = NULL;
-        int             i     = 0;
+        int     ret = -ENOMEM;
 
-        GF_ASSERT (child_up);
-        GF_ASSERT (call_child);
-        GF_ASSERT (last_index);
-        GF_ASSERT (fresh_children);
-        GF_ASSERT (read_child >= 0);
+        local->xattr_req = dict_new ();
+        if (!local->xattr_req)
+                goto out;
+        if (xattr_req)
+                dict_copy (xattr_req, local->xattr_req);
 
-        priv = this->private;
-        *call_child = -1;
-        *last_index = -1;
+        ret = afr_xattr_req_prepare (this, local->xattr_req);
+	if (ret < 0) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"%s: Unable to prepare xattr_req", loc->path);
+	}
 
-        if (child_up[read_child]) {
-                *call_child = read_child;
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (fresh_children[i] == -1)
-                                break;
-                        if (child_up[fresh_children[i]]) {
-                                *call_child = fresh_children[i];
-                                ret = 0;
-                                break;
-                        }
-                }
-
-                if (*call_child == -1) {
-                        ret = -ENOTCONN;
-                        goto out;
-                }
+        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_INODELK_COUNT);
+        }
+        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
+        }
 
-                *last_index = i;
+        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
         }
+
+        ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d, call_child: %d, "
-                "last_index: %d", ret, *call_child, *last_index);
         return ret;
 }
 
-void
-afr_reset_xattr (dict_t **xattr, unsigned int child_count)
+
+int
+afr_hash_child (inode_t *inode, int32_t child_count, int hashmode)
 {
-        unsigned int i = 0;
+        uuid_t gfid_copy = {0,};
+        pid_t pid;
 
-        if (!xattr)
-                goto out;
-        for (i = 0; i < child_count; i++) {
-                if (xattr[i]) {
-                        dict_unref (xattr[i]);
-                        xattr[i] = NULL;
-                }
+        if (!hashmode) {
+                return -1;
         }
-out:
-        return;
+
+        if (inode) {
+               uuid_copy (gfid_copy, inode->gfid);
+        }
+
+        if (hashmode > 1) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and returns a
+                 * constant-length value that's sure to be shorter than a UUID.
+                 * It's still very unlikely to be the same across clients, so
+                 * it still provides good mixing.  We're not trying for
+                 * perfection here.  All we need is a low probability that
+                 * multiple clients won't converge on the same subvolume.
+                 */
+                pid = getpid();
+                memcpy (gfid_copy, &pid, sizeof(pid));
+        }
+
+        return SuperFastHash((char *)gfid_copy,
+                             sizeof(gfid_copy)) % child_count;
 }
 
-void
-afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
-{
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
 
-        sh = &local->self_heal;
-        priv = this->private;
+int
+afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
+				  unsigned char *readable)
+{
+	afr_private_t *priv = NULL;
+	int read_subvol = -1;
+	int i = 0;
 
-        if (sh->buf)
-                GF_FREE (sh->buf);
+	priv = this->private;
 
-        if (sh->parentbufs)
-                GF_FREE (sh->parentbufs);
+	/* first preference - explicitly specified or local subvolume */
+	if (priv->read_child >= 0 && readable[priv->read_child])
+		return priv->read_child;
 
-        if (sh->inode)
-                inode_unref (sh->inode);
+	/* second preference - use hashed mode */
+	read_subvol = afr_hash_child (inode, priv->child_count,
+				      priv->hash_mode);
+	if (read_subvol >= 0 && readable[read_subvol])
+		return read_subvol;
 
-        if (sh->xattr) {
-                afr_reset_xattr (sh->xattr, priv->child_count);
-                GF_FREE (sh->xattr);
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (readable[i])
+			return i;
+	}
 
-        if (sh->child_errno)
-                GF_FREE (sh->child_errno);
+	/* no readable subvolumes, either split brain or all subvols down */
 
-        afr_matrix_cleanup (sh->pending_matrix, priv->child_count);
-        afr_matrix_cleanup (sh->delta_matrix, priv->child_count);
+	return -1;
+}
 
-        if (sh->sources)
-                GF_FREE (sh->sources);
 
-        if (sh->success)
-                GF_FREE (sh->success);
+int
+afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
+				unsigned char *readable, int *event_p,
+				int type)
+{
+	int ret = -1;
 
-        if (sh->locked_nodes)
-                GF_FREE (sh->locked_nodes);
+	if (type == AFR_METADATA_TRANSACTION)
+		ret = afr_inode_read_subvol_get (inode, this, 0, readable,
+						 event_p);
+	else
+		ret = afr_inode_read_subvol_get (inode, this, readable, 0,
+						 event_p);
+	return ret;
+}
 
-        if (sh->healing_fd) {
-                fd_unref (sh->healing_fd);
-                sh->healing_fd = NULL;
-        }
 
-        if (sh->linkname)
-                GF_FREE ((char *)sh->linkname);
+int
+afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
+		     int *event_p, afr_transaction_type type)
+{
+	afr_private_t *priv = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	unsigned char *readable = NULL;
+	unsigned char *intersection = NULL;
+	int subvol = -1;
+	int event = 0;
 
-        if (sh->success_children)
-                GF_FREE (sh->success_children);
+	priv = this->private;
 
-        if (sh->fresh_children)
-                GF_FREE (sh->fresh_children);
+	readable = alloca0 (priv->child_count);
+	data_readable = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+	intersection = alloca0 (priv->child_count);
 
-        if (sh->fresh_parent_dirs)
-                GF_FREE (sh->fresh_parent_dirs);
+	afr_inode_read_subvol_type_get (inode, this, readable, &event, type);
 
-        loc_wipe (&sh->parent_loc);
-        loc_wipe (&sh->lookup_loc);
+	afr_inode_read_subvol_get (inode, this, data_readable, metadata_readable,
+				   &event);
 
-        if (sh->checksum)
-                GF_FREE (sh->checksum);
+	AFR_INTERSECT (intersection, data_readable, metadata_readable,
+		       priv->child_count);
 
-        if (sh->write_needed)
-                GF_FREE (sh->write_needed);
-        if (sh->healing_fd)
-                fd_unref (sh->healing_fd);
+	if (AFR_COUNT (intersection, priv->child_count) > 0)
+		subvol = afr_read_subvol_select_by_policy (inode, this,
+							   intersection);
+	else
+		subvol = afr_read_subvol_select_by_policy (inode, this,
+							   readable);
+	if (subvol_p)
+		*subvol_p = subvol;
+	if (event_p)
+		*event_p = event;
+	return subvol;
 }
 
 
 void
 afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 {
-        afr_private_t * priv = NULL;
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
 
         priv = this->private;
 
         afr_matrix_cleanup (local->pending, priv->child_count);
 
-        if (local->internal_lock.locked_nodes)
-                GF_FREE (local->internal_lock.locked_nodes);
+        GF_FREE (local->internal_lock.locked_nodes);
 
-        if (local->internal_lock.inode_locked_nodes)
-                GF_FREE (local->internal_lock.inode_locked_nodes);
-
-        if (local->internal_lock.entry_locked_nodes)
-                GF_FREE (local->internal_lock.entry_locked_nodes);
+        for (i = 0; local->internal_lock.inodelk[i].domain; i++) {
+                GF_FREE (local->internal_lock.inodelk[i].locked_nodes);
+        }
 
-        if (local->internal_lock.lower_locked_nodes)
-                GF_FREE (local->internal_lock.lower_locked_nodes);
+        GF_FREE (local->internal_lock.lower_locked_nodes);
 
+        afr_entry_lockee_cleanup (&local->internal_lock);
 
         GF_FREE (local->transaction.pre_op);
-        GF_FREE (local->transaction.child_errno);
         GF_FREE (local->transaction.eager_lock);
 
         GF_FREE (local->transaction.basename);
@@ -815,10 +867,39 @@ afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 
         loc_wipe (&local->transaction.parent_loc);
         loc_wipe (&local->transaction.new_parent_loc);
+
 }
 
 
 void
+afr_replies_wipe (afr_local_t *local, afr_private_t *priv)
+{
+	int i;
+
+	if (!local->replies)
+		return;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].xdata) {
+			dict_unref (local->replies[i].xdata);
+			local->replies[i].xdata = NULL;
+		}
+	}
+
+	memset (local->replies, 0, sizeof(*local->replies) * priv->child_count);
+}
+
+void
+afr_remove_eager_lock_stub (afr_local_t *local)
+{
+        LOCK (&local->fd->lock);
+        {
+                list_del_init (&local->transaction.eager_locked);
+        }
+        UNLOCK (&local->fd->lock);
+}
+
+void
 afr_local_cleanup (afr_local_t *local, xlator_t *this)
 {
         afr_private_t * priv = NULL;
@@ -826,7 +907,11 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (!local)
                 return;
 
-        afr_local_sh_cleanup (local, this);
+	syncbarrier_destroy (&local->barrier);
+
+        if (local->transaction.eager_lock_on &&
+            !list_empty (&local->transaction.eager_locked))
+                afr_remove_eager_lock_stub (local);
 
         afr_local_transaction_cleanup (local, this);
 
@@ -841,55 +926,36 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (local->xattr_req)
                 dict_unref (local->xattr_req);
 
-        if (local->child_up)
-                GF_FREE (local->child_up);
-
-        if (local->child_errno)
-                GF_FREE (local->child_errno);
+        if (local->dict)
+                dict_unref (local->dict);
 
-        if (local->fresh_children)
-                GF_FREE (local->fresh_children);
+	afr_replies_wipe (local, priv);
+	GF_FREE(local->replies);
 
-        if (local->fd_open_on)
-                GF_FREE (local->fd_open_on);
+        GF_FREE (local->child_up);
 
-        { /* lookup */
-                if (local->cont.lookup.xattrs) {
-                        afr_reset_xattr (local->cont.lookup.xattrs,
-                                         priv->child_count);
-                        GF_FREE (local->cont.lookup.xattrs);
-                        local->cont.lookup.xattrs = NULL;
-                }
+        GF_FREE (local->read_attempted);
 
-                if (local->cont.lookup.xattr) {
-                        dict_unref (local->cont.lookup.xattr);
-                }
+        GF_FREE (local->readable);
 
-                if (local->cont.lookup.inode) {
-                        inode_unref (local->cont.lookup.inode);
-                }
+	if (local->inode)
+		inode_unref (local->inode);
 
-                if (local->cont.lookup.postparents)
-                        GF_FREE (local->cont.lookup.postparents);
+	if (local->parent)
+		inode_unref (local->parent);
 
-                if (local->cont.lookup.bufs)
-                        GF_FREE (local->cont.lookup.bufs);
+	if (local->parent2)
+		inode_unref (local->parent2);
 
-                if (local->cont.lookup.success_children)
-                        GF_FREE (local->cont.lookup.success_children);
-
-                if (local->cont.lookup.sources)
-                        GF_FREE (local->cont.lookup.sources);
-        }
+	if (local->refreshinode)
+		inode_unref (local->refreshinode);
 
         { /* getxattr */
-                if (local->cont.getxattr.name)
-                        GF_FREE (local->cont.getxattr.name);
+                GF_FREE (local->cont.getxattr.name);
         }
 
         { /* lk */
-                if (local->cont.lk.locked_nodes)
-                        GF_FREE (local->cont.lk.locked_nodes);
+                GF_FREE (local->cont.lk.locked_nodes);
         }
 
         { /* create */
@@ -931,20 +997,32 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         { /* removexattr */
                 GF_FREE (local->cont.removexattr.name);
         }
-
+        { /* xattrop */
+                if (local->cont.xattrop.xattr)
+                        dict_unref (local->cont.xattrop.xattr);
+        }
+        { /* fxattrop */
+                if (local->cont.fxattrop.xattr)
+                        dict_unref (local->cont.fxattrop.xattr);
+        }
         { /* symlink */
                 GF_FREE (local->cont.symlink.linkpath);
         }
 
         { /* opendir */
-                if (local->cont.opendir.checksum)
-                        GF_FREE (local->cont.opendir.checksum);
+                GF_FREE (local->cont.opendir.checksum);
         }
 
         { /* readdirp */
                 if (local->cont.readdir.dict)
                         dict_unref (local->cont.readdir.dict);
         }
+
+        if (local->xdata_req)
+                dict_unref (local->xdata_req);
+
+        if (local->xdata_rsp)
+                dict_unref (local->xdata_rsp);
 }
 
 
@@ -965,1122 +1043,880 @@ afr_frame_return (call_frame_t *frame)
         return call_count;
 }
 
-int
-afr_set_elem_count_get (unsigned char *elems, int child_count)
-{
-        int i   = 0;
-        int ret = 0;
-
-        for (i = 0; i < child_count; i++)
-                if (elems[i])
-                        ret++;
-        return ret;
-}
-
-/**
- * up_children_count - return the number of children that are up
- */
 
-unsigned int
-afr_up_children_count (unsigned char *child_up, unsigned int child_count)
+gf_boolean_t
+afr_is_entry_possibly_under_txn (afr_local_t *local, xlator_t *this)
 {
-        return afr_set_elem_count_get (child_up, child_count);
-}
+	int i = 0;
+	int tmp = 0;
+	afr_private_t *priv = NULL;
 
-unsigned int
-afr_locked_children_count (unsigned char *children, unsigned int child_count)
-{
-        return afr_set_elem_count_get (children, child_count);
-}
+	priv = this->private;
 
-unsigned int
-afr_pre_op_done_children_count (unsigned char *pre_op,
-                                unsigned int child_count)
-{
-        return afr_set_elem_count_get (pre_op, child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].xdata)
+			continue;
+		if (dict_get_int32 (local->replies[i].xdata,
+				    GLUSTERFS_PARENT_ENTRYLK,
+				    &tmp) == 0)
+			if (tmp)
+				return _gf_true;
+	}
+
+	return _gf_false;
 }
 
-gf_boolean_t
-afr_is_fresh_lookup (loc_t *loc, xlator_t *this)
-{
-        uint64_t          ctx = 0;
-        int32_t           ret = 0;
 
-        GF_ASSERT (loc);
-        GF_ASSERT (this);
-        GF_ASSERT (loc->inode);
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
 
-        ret = inode_ctx_get (loc->inode, this, &ctx);
-        if (0 == ret)
-                return _gf_false;
-        return _gf_true;
+static void
+afr_handle_quota_size (call_frame_t *frame, xlator_t *this)
+{
+	unsigned char *readable = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	struct afr_reply *replies = NULL;
+	int i = 0;
+	uint64_t size = 0;
+	uint64_t max_size = 0;
+	int readable_cnt = 0;
+
+	local = frame->local;
+	priv = this->private;
+	replies = local->replies;
+
+	readable = alloca0 (priv->child_count);
+
+	afr_inode_read_subvol_get (local->inode, this, readable, 0, 0);
+
+	readable_cnt = AFR_COUNT (readable, priv->child_count);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1)
+			continue;
+		if (readable_cnt && !readable[i])
+			continue;
+		if (!replies[i].xdata)
+			continue;
+		if (dict_get_uint64 (replies[i].xdata, QUOTA_SIZE_KEY, &size))
+			continue;
+		if (size > max_size)
+			max_size = size;
+	}
+
+	if (!max_size)
+		return;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1)
+			continue;
+		if (readable_cnt && !readable[i])
+			continue;
+		if (!replies[i].xdata)
+			continue;
+		if (dict_set_uint64 (replies[i].xdata, QUOTA_SIZE_KEY, max_size))
+			continue;
+	}
 }
 
-void
-afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
+
+static void
+afr_lookup_done (call_frame_t *frame, xlator_t *this)
 {
-        GF_ASSERT (loc);
-        GF_ASSERT (buf);
+        afr_private_t       *priv  = NULL;
+        afr_local_t         *local = NULL;
+	int                 i = -1;
+	int                 op_errno = 0;
+	int                 read_subvol = 0;
+	unsigned char      *readable = NULL;
+	int                 event = 0;
+	struct afr_reply   *replies = NULL;
+	uuid_t              read_gfid = {0, };
+	gf_boolean_t        locked_entry = _gf_false;
+	gf_boolean_t        can_interpret = _gf_true;
 
-        uuid_copy (loc->gfid, buf->ia_gfid);
-        if (postparent)
-                uuid_copy (loc->pargfid, postparent->ia_gfid);
-}
+        priv  = this->private;
+        local = frame->local;
+	replies = local->replies;
 
-int
-afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
-{
-        struct iatt     *buf = NULL;
-        struct iatt     *postparent = NULL;
-        dict_t          **xattr = NULL;
-        int32_t         *success_children = NULL;
-        int32_t         *sources = NULL;
-        afr_private_t   *priv = NULL;
-        int32_t         read_child = -1;
-        int             ret = 0;
-        int             i = 0;
+	locked_entry = afr_is_entry_possibly_under_txn (local, this);
 
-        GF_ASSERT (local);
+	readable = alloca0 (priv->child_count);
 
-        buf = &local->cont.lookup.buf;
-        postparent = &local->cont.lookup.postparent;
-        xattr = &local->cont.lookup.xattr;
-        priv = this->private;
+	afr_inode_read_subvol_get (local->loc.parent, this, readable,
+				   NULL, &event);
 
-        read_child = afr_inode_get_read_ctx (this, local->cont.lookup.inode,
-                                             local->fresh_children);
-        if (read_child < 0) {
-                ret = -1;
-                goto out;
-        }
-        success_children = local->cont.lookup.success_children;
-        sources = local->cont.lookup.sources;
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-        afr_children_intersection_get (local->fresh_children, success_children,
-                                       sources, priv->child_count);
-        if (!sources[read_child]) {
-                read_child = -1;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sources[i]) {
-                                read_child = i;
-                                break;
-                        }
-                }
-        }
-        if (read_child < 0) {
-                ret = -1;
-                goto out;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Building lookup response from %d",
-                read_child);
-        if (!*xattr)
-                *xattr = dict_ref (local->cont.lookup.xattrs[read_child]);
-        *buf = local->cont.lookup.bufs[read_child];
-        *postparent = local->cont.lookup.postparents[read_child];
-
-        if (IA_INVAL == local->cont.lookup.inode->ia_type) {
-                /* fix for RT #602 */
-                local->cont.lookup.inode->ia_type = buf->ia_type;
-        }
-out:
-        return ret;
+	/* First, check if we have an ESTALE from somewhere,
+	   If so, propagate that so that a revalidate can be
+	   issued
+	*/
+	op_errno = afr_final_errno (frame->local, this->private);
+	local->op_errno = op_errno;
+        if (op_errno == ESTALE) {
+		local->op_errno = op_errno;
+		local->op_ret = -1;
+                goto unwind;
+	}
+
+	read_subvol = -1;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (locked_entry && replies[i].op_ret == -1 &&
+		    replies[i].op_errno == ENOENT) {
+			/* Second, check entry is still
+			   "underway" in creation */
+			local->op_ret = -1;
+			local->op_errno = ENOENT;
+			read_subvol = i;
+			goto unwind;
+		}
+
+		if (replies[i].op_ret == -1)
+			continue;
+
+		if (read_subvol == -1 || !readable[read_subvol]) {
+			read_subvol = i;
+			uuid_copy (read_gfid, replies[i].poststat.ia_gfid);
+			local->op_ret = 0;
+		}
+	}
+
+	if (read_subvol == -1)
+		goto unwind;
+	/* We now have a read_subvol, which is readable[] (if there
+	   were any). Next we look for GFID mismatches. We don't
+	   consider a GFID mismatch as an error if read_subvol is
+	   readable[] but the mismatching GFID subvol is not.
+	*/
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1) {
+			if (priv->child_up[i])
+				can_interpret = _gf_false;
+			continue;
+		}
+
+		if (!uuid_compare (replies[i].poststat.ia_gfid,
+				   read_gfid))
+			continue;
+
+		can_interpret = _gf_false;
+
+		if (locked_entry)
+			continue;
+
+		/* Now GFIDs mismatch. It's OK as long as this subvol
+		   is not readable[] but read_subvol is */
+		if (readable[read_subvol] && !readable[i])
+			continue;
+
+		/* LOG ERROR */
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto unwind;
+	}
+
+	/* Forth, for the finalized GFID, pick the best subvolume
+	   to return stats from.
+	*/
+	if (can_interpret) {
+		/* It is safe to call afr_replies_interpret() because we have
+		   a response from all the UP subvolumes and all of them resolved
+		   to the same GFID
+		*/
+		if (afr_replies_interpret (frame, this, local->inode)) {
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   0, 0);
+			afr_inode_read_subvol_reset (local->inode, this);
+			goto cant_interpret;
+		} else {
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   0, 0);
+		}
+	} else {
+	cant_interpret:
+		if (read_subvol == -1)
+			dict_del (replies[0].xdata, GF_CONTENT_KEY);
+		else
+			dict_del (replies[read_subvol].xdata, GF_CONTENT_KEY);
+	}
+
+	afr_handle_quota_size (frame, this);
+
+unwind:
+	if (read_subvol == -1)
+		read_subvol = 0;
+
+	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+			  local->inode, &local->replies[read_subvol].poststat,
+			  local->replies[read_subvol].xdata,
+			  &local->replies[read_subvol].postparent);
 }
 
-static void
-afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
-                            int child_index, dict_t *xattr)
-{
-        uint32_t inodelk_count = 0;
-        uint32_t entrylk_count = 0;
-        int      ret           = -1;
-
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (xattr);
-        GF_ASSERT (child_index >= 0);
+/*
+ * During a lookup, some errors are more "important" than
+ * others in that they must be given higher priority while
+ * returning to the user.
+ *
+ * The hierarchy is ESTALE > ENOENT > others
+ */
 
-        ret = dict_get_uint32 (xattr, GLUSTERFS_INODELK_COUNT,
-                               &inodelk_count);
-        if (ret == 0)
-                local->inodelk_count += inodelk_count;
+int
+afr_higher_errno (int32_t old_errno, int32_t new_errno)
+{
+	if (old_errno == ENODATA || new_errno == ENODATA)
+		return ENODATA;
+	if (old_errno == ESTALE || new_errno == ESTALE)
+		return ESTALE;
+	if (old_errno == ENOENT || new_errno == ENOENT)
+		return ENOENT;
 
-        ret = dict_get_uint32 (xattr, GLUSTERFS_ENTRYLK_COUNT,
-                               &entrylk_count);
-        if (ret == 0)
-                local->entrylk_count += entrylk_count;
+	return new_errno;
 }
 
-static void
-afr_lookup_set_self_heal_params_by_xattr (afr_local_t *local, xlator_t *this,
-                                          dict_t *xattr)
-{
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (xattr);
 
-        if (afr_sh_has_metadata_pending (xattr, this)) {
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "metadata self-heal is pending for %s.",
-                       local->loc.path);
-        }
+int
+afr_final_errno (afr_local_t *local, afr_private_t *priv)
+{
+	int i = 0;
+	int op_errno = 0;
+	int tmp_errno = 0;
 
-        if (afr_sh_has_entry_pending (xattr, this)) {
-                local->self_heal.do_entry_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "entry self-heal is pending for %s.", local->loc.path);
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret == 0)
+			continue;
+		tmp_errno = local->replies[i].op_errno;
+		op_errno = afr_higher_errno (op_errno, tmp_errno);
+	}
 
-        if (afr_sh_has_data_pending (xattr, this)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "data self-heal is pending for %s.", local->loc.path);
-        }
+	return op_errno;
 }
 
-static void
-afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
-                            struct iatt *buf, struct iatt *lookup_buf)
+static int
+get_pathinfo_host (char *pathinfo, char *hostname, size_t size)
 {
-        if (PERMISSION_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                gf_log (this->name, GF_LOG_INFO,
-                        "permissions differ for %s ", local->loc.path);
-                local->self_heal.do_metadata_self_heal = _gf_true;
-        }
+        char    *start = NULL;
+        char    *end = NULL;
+        int     ret  = -1;
+        int     i    = 0;
 
-        if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log (this->name, GF_LOG_INFO,
-                        "ownership differs for %s ", local->loc.path);
-        }
+        if (!pathinfo)
+                goto out;
 
-        if (SIZE_DIFFERS (buf, lookup_buf)
-            && IA_ISREG (buf->ia_type)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "size differs for %s ", local->loc.path);
-                local->self_heal.do_data_self_heal = _gf_true;
-        }
+        start = strchr (pathinfo, ':');
+        if (!start)
+                goto out;
+        end = strrchr (pathinfo, ':');
+        if (start == end)
+                goto out;
 
-        if (uuid_compare (buf->ia_gfid, lookup_buf->ia_gfid)) {
-                /* mismatching gfid */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: gfid different on subvolume", local->loc.path);
-        }
+        memset (hostname, 0, size);
+        i = 0;
+        while (++start != end)
+                hostname[i++] = *start;
+        ret = 0;
+out:
+        return ret;
 }
 
-static void
-afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this,
-                                       gf_boolean_t split_brain)
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *local)
 {
-        GF_ASSERT (local);
-        GF_ASSERT (this);
+        int             ret   = 0;
+        char            pathinfohost[1024] = {0};
+        char            localhost[1024] = {0};
+        xlator_t        *this = THIS;
 
-        if ((local->success_count > 0) && (local->enoent_count > 0)) {
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                local->self_heal.do_data_self_heal     = _gf_true;
-                local->self_heal.do_entry_self_heal    = _gf_true;
-                local->self_heal.do_gfid_self_heal    = _gf_true;
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-                gf_log(this->name, GF_LOG_INFO,
-                       "entries are missing in lookup of %s.",
-                       local->loc.path);
-                //If all self-heals are needed no need to check for other rules
+        *local = _gf_false;
+        ret = get_pathinfo_host (pathinfo, pathinfohost, sizeof (pathinfohost));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid pathinfo: %s",
+                        pathinfo);
                 goto out;
         }
 
-        if ((local->success_count > 0) && split_brain &&
-            IA_ISREG (local->cont.lookup.inode->ia_type)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                local->self_heal.do_gfid_self_heal    = _gf_true;
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "split brain detected during lookup of %s.",
-                        local->loc.path);
+        ret = gethostname (localhost, sizeof (localhost));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "gethostname() failed, "
+                        "reason: %s", strerror (errno));
+                goto out;
         }
 
+        if (!strcmp (localhost, pathinfohost))
+                *local = _gf_true;
 out:
-        return;
-}
-
-gf_boolean_t
-afr_can_self_heal_proceed (afr_self_heal_t *sh, afr_private_t *priv)
-{
-        GF_ASSERT (sh);
-        GF_ASSERT (priv);
-
-        return (sh->do_gfid_self_heal
-                || sh->do_missing_entry_self_heal
-                || (afr_data_self_heal_enabled (priv->data_self_heal) &&
-                    sh->do_data_self_heal)
-                || (priv->metadata_self_heal && sh->do_metadata_self_heal)
-                || (priv->entry_self_heal && sh->do_entry_self_heal));
+        return ret;
 }
 
-afr_transaction_type
-afr_transaction_type_get (ia_type_t ia_type)
+static int32_t
+afr_local_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			 int32_t op_ret, int32_t op_errno, dict_t *dict,
+			 dict_t *xdata)
 {
-        afr_transaction_type    type = AFR_METADATA_TRANSACTION;
-
-        GF_ASSERT (ia_type != IA_INVAL);
+        int              ret            = 0;
+        char            *pathinfo       = NULL;
+        gf_boolean_t     is_local        = _gf_false;
+        afr_private_t   *priv           = NULL;
+        int32_t          child_index    = -1;
 
-        if (IA_ISDIR (ia_type)) {
-                type = AFR_ENTRY_TRANSACTION;
-        } else if (IA_ISREG (ia_type)) {
-                type = AFR_DATA_TRANSACTION;
+        if (op_ret != 0) {
+                goto out;
         }
-        return type;
-}
 
-int
-afr_lookup_select_read_child (afr_local_t *local, xlator_t *this,
-                              int32_t *read_child)
-{
-        ia_type_t               ia_type        = IA_INVAL;
-        int32_t                 source         = -1;
-        int                     ret            = -1;
-        dict_t                  **xattrs       = NULL;
-        int32_t                 *success_children = NULL;
-        afr_transaction_type    type           = AFR_METADATA_TRANSACTION;
+	priv = this->private;
+	child_index = (int32_t)(long)cookie;
 
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (local->success_count > 0);
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        if (ret != 0) {
+                goto out;
+        }
 
-        success_children = local->cont.lookup.success_children;
-        /*We can take the success_children[0] only because we already
-         *handle the conflicting children other wise, we could select the
-         *read_child based on wrong file type
-         */
-        ia_type = local->cont.lookup.bufs[success_children[0]].ia_type;
-        type = afr_transaction_type_get (ia_type);
-        xattrs = local->cont.lookup.xattrs;
-        source = afr_lookup_select_read_child_by_txn_type (this, local, xattrs,
-                                                           type);
-        if (source < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to select source "
-                        "for %s", local->loc.path);
+        ret = afr_local_pathinfo (pathinfo, &is_local);
+        if (ret) {
                 goto out;
         }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Source selected as %d for %s",
-                source, local->loc.path);
-        *read_child = source;
-        ret = 0;
+        /*
+         * Note that one local subvolume will override another here.  The only
+         * way to avoid that would be to retain extra information about whether
+         * the previous read_child is local, and it's just not worth it.  Even
+         * the slowest local subvolume is far preferable to a remote one.
+         */
+        if (is_local) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "selecting local read_child %s",
+                        priv->children[child_index]->name);
+                priv->read_child = child_index;
+        }
 out:
-        return ret;
+        STACK_DESTROY(frame->root);
+        return 0;
 }
 
-static inline gf_boolean_t
-afr_is_transaction_running (afr_local_t *local)
+static void
+afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
 {
-        GF_ASSERT (local->fop == GF_FOP_LOOKUP);
-        return ((local->inodelk_count > 0) || (local->entrylk_count > 0));
-}
+        call_frame_t    *newframe = NULL;
+        loc_t            tmploc = {0,};
+        afr_private_t   *priv = this->private;
 
-void
-afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                      gf_boolean_t background, ia_type_t ia_type, char *reason,
-                      void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
-                                                   xlator_t *this),
-                      int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno))
-{
-        afr_local_t             *local = NULL;
-        char                    sh_type_str[256] = {0,};
-        char                    *bg = "";
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-        GF_ASSERT (ia_type != IA_INVAL);
-
-        local = frame->local;
-        local->self_heal.background = background;
-        local->self_heal.type       = ia_type;
-        local->self_heal.unwind     = unwind;
-        local->self_heal.gfid_sh_success_cbk     = gfid_sh_success_cbk;
-
-        afr_self_heal_type_str_get (&local->self_heal,
-                                    sh_type_str,
-                                    sizeof (sh_type_str));
-
-        if (background)
-                bg = "background";
-        gf_log (this->name, GF_LOG_INFO,
-                "%s %s self-heal triggered. path: %s, reason: %s", bg,
-                sh_type_str, local->loc.path, reason);
+        newframe = create_frame(this,this->ctx->pool);
+        if (!newframe) {
+                return;
+        }
 
-        afr_self_heal (frame, this, inode);
+        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
+        STACK_WIND_COOKIE (newframe, afr_local_discovery_cbk,
+                           (void *)(long)child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->getxattr,
+                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
 }
 
-unsigned int
-afr_gfid_missing_count (const char *xlator_name, int32_t *success_children,
-                        struct iatt *bufs, unsigned int child_count,
-                        const char *path)
-{
-        unsigned int    gfid_miss_count   = 0;
-        int             i              = 0;
-        struct iatt     *child1        = NULL;
 
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                child1 = &bufs[success_children[i]];
-                if (uuid_is_null (child1->ia_gfid)) {
-                        gf_log (xlator_name, GF_LOG_DEBUG, "%s: gfid is null"
-                                " on subvolume %d", path, success_children[i]);
-                        gfid_miss_count++;
-                }
-        }
+int
+afr_lookup_selfheal_wrap (void *opaque)
+{
+	call_frame_t *frame = opaque;
+	afr_local_t *local = NULL;
+	xlator_t *this = NULL;
+	inode_t *inode = NULL;
 
-        return gfid_miss_count;
-}
+	local = frame->local;
+	this = frame->this;
 
-static int
-afr_lookup_gfid_missing_count (afr_local_t *local, xlator_t *this)
-{
-        int32_t         *success_children = NULL;
-        afr_private_t   *priv          = NULL;
-        struct iatt     *bufs          = NULL;
-        int             miss_count     = 0;
+	afr_selfheal_name (frame->this, local->loc.pargfid, local->loc.name);
 
-        priv = this->private;
-        bufs = local->cont.lookup.bufs;
-        success_children = local->cont.lookup.success_children;
+	afr_replies_wipe (local, this->private);
 
-        miss_count =  afr_gfid_missing_count (this->name, success_children,
-                                              bufs, priv->child_count,
-                                              local->loc.path);
-        return miss_count;
-}
+	inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
+						 local->loc.name, local->replies,
+						 local->child_up);
+	if (inode)
+		inode_unref (inode);
+	afr_lookup_done (frame, this);
 
-gf_boolean_t
-afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
-                        unsigned int child_count, const char *path,
-                        const char *xlator_name)
-{
-        gf_boolean_t    conflicting    = _gf_false;
-        int             i              = 0;
-        struct iatt     *child1        = NULL;
-        struct iatt     *child2        = NULL;
-        uuid_t          *gfid          = NULL;
-
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                child1 = &bufs[success_children[i]];
-                if ((!gfid) && (!uuid_is_null (child1->ia_gfid)))
-                        gfid = &child1->ia_gfid;
-
-                if (i == 0)
-                        continue;
-
-                child2 = &bufs[success_children[i-1]];
-                if (FILETYPE_DIFFERS (child1, child2)) {
-                        gf_log (xlator_name, GF_LOG_WARNING, "%s: filetype "
-                                "differs on subvolumes (%d, %d)", path,
-                                success_children[i-1], success_children[i]);
-                        conflicting = _gf_true;
-                        goto out;
-                }
-                if (!gfid || uuid_is_null (child1->ia_gfid))
-                        continue;
-                if (uuid_compare (*gfid, child1->ia_gfid)) {
-                       gf_log (xlator_name, GF_LOG_WARNING, "%s: gfid differs"
-                               " on subvolume %d", path, success_children[i]);
-                       conflicting = _gf_true;
-                       goto out;
-                }
-        }
-out:
-        return conflicting;
+	return 0;
 }
 
-/* afr_update_gfid_from_iatts: This function should be called only if the
- * iatts are not conflicting.
- */
-void
-afr_update_gfid_from_iatts (uuid_t uuid, struct iatt *bufs,
-                            int32_t *success_children, unsigned int child_count)
-{
-        uuid_t          *gfid = NULL;
-        int             i = 0;
-        int             child = 0;
 
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child == -1)
-                        break;
-                if ((!gfid) && (!uuid_is_null (bufs[child].ia_gfid))) {
-                        gfid = &bufs[child].ia_gfid;
-                } else if (gfid && (!uuid_is_null (bufs[child].ia_gfid))) {
-                        if (uuid_compare (*gfid, bufs[child].ia_gfid)) {
-                                GF_ASSERT (0);
-                                goto out;
-                        }
-                }
-        }
-        if (gfid && (!uuid_is_null (*gfid)))
-                uuid_copy (uuid, *gfid);
-out:
-        return;
+int
+afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	call_frame_t *heal = NULL;
+	int i = 0, first = -1;
+	gf_boolean_t need_heal = _gf_false;
+	struct afr_reply *replies = NULL;
+	int ret = 0;
+
+	local = frame->local;
+	replies = local->replies;
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first == -1) {
+			first = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first].op_ret) {
+			need_heal = _gf_true;
+			break;
+		}
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first].poststat.ia_gfid)) {
+			need_heal = _gf_true;
+			break;
+		}
+	}
+
+	if (need_heal) {
+		heal = copy_frame (frame);
+		if (heal)
+			heal->root->pid = -1;
+		ret = synctask_new (this->ctx->env, afr_lookup_selfheal_wrap,
+				    afr_refresh_selfheal_done, heal, frame);
+		if (ret)
+			goto lookup_done;
+	} else {
+	lookup_done:
+		afr_lookup_done (frame, this);
+	}
+
+	return ret;
 }
 
-static gf_boolean_t
-afr_lookup_conflicting_entries (afr_local_t *local, xlator_t *this)
+
+int
+afr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+		dict_t *xdata, struct iatt *postparent)
 {
-        afr_private_t           *priv = NULL;
-        gf_boolean_t            conflict = _gf_false;
+        afr_local_t *   local = NULL;
+        int             call_count      = -1;
+        int             child_index     = -1;
 
-        priv = this->private;
-        conflict =  afr_conflicting_iattrs (local->cont.lookup.bufs,
-                                            local->cont.lookup.success_children,
-                                            priv->child_count, local->loc.path,
-                                            this->name);
-        return conflict;
-}
+	child_index = (long) cookie;
 
-gf_boolean_t
-afr_open_only_data_self_heal (char *data_self_heal)
-{
-        return !strcmp (data_self_heal, "open");
-}
+	local = frame->local;
 
-gf_boolean_t
-afr_data_self_heal_enabled (char *data_self_heal)
-{
-        gf_boolean_t    enabled = _gf_false;
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[child_index].poststat = *buf;
+		local->replies[child_index].postparent = *postparent;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	}
 
-        if (gf_string2boolean (data_self_heal, &enabled) == -1) {
-                enabled = !strcmp (data_self_heal, "open");
-                GF_ASSERT (enabled);
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+		afr_lookup_entry_heal (frame, this);
         }
 
-        return enabled;
+	return 0;
 }
 
+
+
 static void
-afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
+afr_discover_done (call_frame_t *frame, xlator_t *this)
 {
-        int                     i = 0;
-        struct iatt             *bufs = NULL;
-        dict_t                  **xattr = NULL;
-        afr_private_t           *priv = NULL;
-        int32_t                 child1 = -1;
-        int32_t                 child2 = -1;
-        afr_self_heal_t         *sh = NULL;
-        gf_boolean_t            split_brain = _gf_false;
+        afr_private_t       *priv  = NULL;
+        afr_local_t         *local = NULL;
+	int                 i = -1;
+	int                 op_errno = 0;
+	int                 read_subvol = 0;
 
         priv  = this->private;
-        sh = &local->self_heal;
-
-        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
-        afr_detect_self_heal_by_lookup_status (local, this, split_brain);
-
-        if (afr_lookup_gfid_missing_count (local, this))
-                local->self_heal.do_gfid_self_heal    = _gf_true;
+        local = frame->local;
 
-        if (_gf_true == afr_lookup_conflicting_entries (local, this))
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-        else
-                afr_update_gfid_from_iatts (local->self_heal.sh_gfid_req,
-                                            local->cont.lookup.bufs,
-                                            local->cont.lookup.success_children,
-                                            priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret == 0)
+			local->op_ret = 0;
+	}
 
-        bufs = local->cont.lookup.bufs;
-        for (i = 1; i < local->success_count; i++) {
-                child1 = local->cont.lookup.success_children[i-1];
-                child2 = local->cont.lookup.success_children[i];
-                afr_detect_self_heal_by_iatt (local, this,
-                                              &bufs[child1], &bufs[child2]);
-        }
+	op_errno = afr_final_errno (frame->local, this->private);
 
-        xattr = local->cont.lookup.xattrs;
-        for (i = 0; i < local->success_count; i++) {
-                child1 = local->cont.lookup.success_children[i];
-                afr_lookup_set_self_heal_params_by_xattr (local, this,
-                                                          xattr[child1]);
-        }
-        if (afr_open_only_data_self_heal (priv->data_self_heal)
-            && !split_brain)
-                sh->do_data_self_heal = _gf_false;
-}
-
-int
-afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t *local = NULL;
+        if (local->op_ret < 0) {
+		local->op_errno = op_errno;
+		local->op_ret = -1;
+                goto unwind;
+	}
 
-        local = frame->local;
+	afr_replies_interpret (frame, this, local->inode);
 
-        if (op_ret == -1) {
-                local->op_ret = -1;
-                if (afr_error_more_important (local->op_errno, op_errno))
-                        local->op_errno = op_errno;
+	read_subvol = afr_data_subvol_get (local->inode, this, 0, 0);
+	if (read_subvol == -1) {
+		gf_log (this->name, GF_LOG_WARNING, "no read subvols for %s",
+			local->loc.path);
 
-                goto out;
-        } else {
-                local->op_ret = 0;
-        }
+		for (i = 0; i < priv->child_count; i++) {
+			if (!local->replies[i].valid ||
+			    local->replies[i].op_ret == -1)
+				continue;
+			read_subvol = i;
+			break;
+		}
+	}
 
-        afr_lookup_done_success_action (frame, this, _gf_true);
-out:
-        AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->cont.lookup.inode, &local->cont.lookup.buf,
-                          local->cont.lookup.xattr,
-                          &local->cont.lookup.postparent);
+unwind:
+	if (read_subvol == -1)
+		read_subvol = 0;
 
-        return 0;
+	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+			  local->inode, &local->replies[read_subvol].poststat,
+			  local->replies[read_subvol].xdata,
+			  &local->replies[read_subvol].postparent);
 }
 
-//TODO: At the moment only lookup needs this, so not doing any checks, in the
-// future we will have to do fop specific operations
-void
-afr_post_gfid_sh_success (call_frame_t *sh_frame, xlator_t *this)
-{
-        afr_local_t             *local = NULL;
-        afr_local_t             *sh_local = NULL;
-        afr_private_t           *priv = NULL;
-        afr_self_heal_t         *sh = NULL;
-        int                     i = 0;
-        struct iatt             *lookup_bufs = NULL;
-        struct iatt             *lookup_parentbufs = NULL;
-
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        local = sh->orig_frame->local;
-        lookup_bufs = local->cont.lookup.bufs;
-        lookup_parentbufs = local->cont.lookup.postparents;
-        priv = this->private;
 
-        memcpy (lookup_bufs, sh->buf, priv->child_count * sizeof (*sh->buf));
-        memcpy (lookup_parentbufs, sh->parentbufs,
-                priv->child_count * sizeof (*sh->parentbufs));
+int
+afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+		  dict_t *xdata, struct iatt *postparent)
+{
+        afr_local_t *   local = NULL;
+        int             call_count      = -1;
+        int             child_index     = -1;
 
-        afr_reset_xattr (local->cont.lookup.xattrs, priv->child_count);
-        if (local->cont.lookup.xattr) {
-                dict_unref (local->cont.lookup.xattr);
-                local->cont.lookup.xattr = NULL;
-        }
+	child_index = (long) cookie;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i])
-                        local->cont.lookup.xattrs[i] = dict_ref (sh->xattr[i]);
-        }
+	local = frame->local;
 
-        afr_reset_children (local->cont.lookup.success_children,
-                            priv->child_count);
-        afr_children_copy (local->cont.lookup.success_children,
-                           sh->fresh_children, priv->child_count);
-}
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[child_index].poststat = *buf;
+		local->replies[child_index].postparent = *postparent;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	}
 
-static void
-afr_lookup_perform_self_heal (call_frame_t *frame, xlator_t *this,
-                              gf_boolean_t *sh_launched)
-{
-        unsigned int         up_count = 0;
-        afr_private_t       *priv    = NULL;
-        afr_local_t         *local   = NULL;
-        char                *reason  = NULL;
-
-        GF_ASSERT (sh_launched);
-        *sh_launched = _gf_false;
-        priv         = this->private;
-        local        = frame->local;
+        if (local->do_discovery && (op_ret == 0))
+                afr_attempt_local_discovery (this, child_index);
 
-        up_count  = afr_up_children_count (local->child_up, priv->child_count);
-        if (up_count == 1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Only 1 child up - do not attempt to detect self heal");
-                goto out;
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+               afr_discover_done (frame, this);
         }
 
-        afr_lookup_set_self_heal_params (local, this);
-        if (afr_can_self_heal_proceed (&local->self_heal, priv)) {
-                if  (afr_is_transaction_running (local))
-                        goto out;
-
-                reason = "lookup detected pending operations";
-                afr_launch_self_heal (frame, this, local->cont.lookup.inode,
-                                      _gf_true, local->cont.lookup.buf.ia_type,
-                                      reason, afr_post_gfid_sh_success,
-                                      afr_self_heal_lookup_unwind);
-                *sh_launched = _gf_true;
-        }
-out:
-        return;
+	return 0;
 }
 
-void
-afr_get_fresh_children (int32_t *success_children, int32_t *sources,
-                        int32_t *fresh_children, unsigned int child_count)
-{
-        unsigned int i = 0;
-        unsigned int j = 0;
 
-        GF_ASSERT (success_children);
-        GF_ASSERT (sources);
-        GF_ASSERT (fresh_children);
-
-        afr_reset_children (fresh_children, child_count);
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                if (afr_is_read_child (success_children, sources, child_count,
-                                       success_children[i])) {
-                        fresh_children[j] = success_children[i];
-                        j++;
-                }
-        }
-}
-
-static int
-afr_lookup_set_read_ctx (afr_local_t *local, xlator_t *this, int32_t read_child)
+int
+afr_discover_do (call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_private_t           *priv = NULL;
+	int ret = 0;
+	int i = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
 
-        GF_ASSERT (read_child >= 0);
+	local = frame->local;
+	priv = this->private;
 
-        priv = this->private;
-        afr_get_fresh_children (local->cont.lookup.success_children,
-                                local->cont.lookup.sources,
-                                local->fresh_children, priv->child_count);
-        afr_inode_set_read_ctx (this, local->cont.lookup.inode, read_child,
-                                local->fresh_children);
+	if (err) {
+		local->op_errno = -err;
+		ret = -1;
+		goto out;
+	}
 
-        return 0;
-}
+	call_count = local->call_count = AFR_COUNT (local->child_up,
+						    priv->child_count);
 
-int
-afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
-                                gf_boolean_t fail_conflict)
-{
-        int32_t             read_child = -1;
-        int32_t             ret        = -1;
-        afr_local_t         *local     = NULL;
-        afr_private_t       *priv      = NULL;
-
-        local   = frame->local;
-        priv    = this->private;
-
-        if (local->loc.parent == NULL)
-                fail_conflict = _gf_true;
-
-        if (afr_conflicting_iattrs (local->cont.lookup.bufs,
-                                     local->cont.lookup.success_children,
-                                     priv->child_count, local->loc.path,
-                                     this->name)) {
-                if (fail_conflict == _gf_false)
-                        ret = 0;
+        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
+					    &local->loc);
+        if (ret) {
+                local->op_errno = -ret;
+		ret = -1;
                 goto out;
         }
 
-        if (!afr_is_transaction_running (local)) {
-                ret = afr_lookup_select_read_child (local, this, &read_child);
-                if (ret)
-                        goto out;
-
-                ret = afr_lookup_set_read_ctx (local, this, read_child);
-                if (ret)
-                        goto out;
-        }
-
-        ret = afr_lookup_build_response_params (local, this);
-        if (ret)
-                goto out;
-        if (afr_is_fresh_lookup (&local->loc, this)) {
-                afr_update_loc_gfids (&local->loc,
-                                      &local->cont.lookup.buf,
-                                      &local->cont.lookup.postparent);
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_discover_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->lookup,
+                                           &local->loc, local->xattr_req);
+                        if (!--call_count)
+                                break;
+                }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret) {
-                local->op_ret = -1;
-                local->op_errno = EIO;
-        }
-        return ret;
+	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+	return 0;
 }
 
-static void
-afr_lookup_done (call_frame_t *frame, xlator_t *this)
+
+int
+afr_discover (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        int                 unwind = 1;
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-        int                 ret = -1;
-        gf_boolean_t        sh_launched = _gf_false;
-        gf_boolean_t        fail_conflict = _gf_false;
-        int                 gfid_miss_count = 0;
-        int                 enotconn_count = 0;
-        int                 up_children_count = 0;
+	int op_errno = ENOMEM;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	int event = 0;
 
-        priv  = this->private;
-        local = frame->local;
+	priv = this->private;
 
-        if (local->op_ret < 0)
-                goto unwind;
-        gfid_miss_count = afr_lookup_gfid_missing_count (local, this);
-        up_children_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        enotconn_count = priv->child_count - up_children_count;
-        if ((gfid_miss_count == local->success_count) &&
-            (enotconn_count > 0)) {
-                local->op_ret = -1;
-                local->op_errno = EIO;
-                gf_log (this->name, GF_LOG_ERROR, "Failing lookup for %s, "
-                        "LOOKUP on a file without gfid is not allowed when "
-                        "some of the children are down", local->loc.path);
-                goto unwind;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        if ((gfid_miss_count == local->success_count) &&
-            uuid_is_null (local->cont.lookup.gfid_req)) {
-                local->op_ret = -1;
-                local->op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "%s: No gfid present",
-                        local->loc.path);
-                goto unwind;
+        if (!local->call_count) {
+                op_errno = ENOTCONN;
+                goto out;
         }
 
-        if (gfid_miss_count && uuid_is_null (local->cont.lookup.gfid_req))
-                fail_conflict = _gf_true;
-        ret = afr_lookup_done_success_action (frame, this, fail_conflict);
-        if (ret)
-                goto unwind;
-        uuid_copy (local->self_heal.sh_gfid_req, local->cont.lookup.gfid_req);
+	if (__is_root_gfid (loc->inode->gfid)) {
+		if (!this->itable)
+			this->itable = loc->inode->table;
+		if (!priv->root_inode)
+			priv->root_inode = inode_ref (loc->inode);
 
-        afr_lookup_perform_self_heal (frame, this, &sh_launched);
-        if (sh_launched) {
-                unwind = 0;
-                goto unwind;
-        }
+		if (priv->choose_local && !priv->did_discovery) {
+			/* Logic to detect which subvolumes of AFR are
+			   local, in order to prefer them for reads
+			*/
+			local->do_discovery = _gf_true;
+                        priv->did_discovery = _gf_true;
+                }
+	}
 
- unwind:
-         if (unwind) {
-                 AFR_STACK_UNWIND (lookup, frame, local->op_ret,
-                                   local->op_errno, local->cont.lookup.inode,
-                                   &local->cont.lookup.buf,
-                                   local->cont.lookup.xattr,
-                                   &local->cont.lookup.postparent);
-        }
-}
+        local->op = GF_FOP_LOOKUP;
 
-/*
- * During a lookup, some errors are more "important" than
- * others in that they must be given higher priority while
- * returning to the user.
- *
- * The hierarchy is ESTALE > ENOENT > others
- *
- */
+        loc_copy (&local->loc, loc);
 
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno)
-{
-        gf_boolean_t ret = _gf_true;
+	local->inode = inode_ref (loc->inode);
 
-        /* Nothing should ever overwrite ESTALE */
-        if (old_errno == ESTALE)
-                ret = _gf_false;
+	if (xattr_req)
+		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+		   allocate one for us */
+		local->xattr_req = dict_ref (xattr_req);
 
-        /* Nothing should overwrite ENOENT, except ESTALE/EIO*/
-        else if ((old_errno == ENOENT) && (new_errno != ESTALE)
-                 && (new_errno != EIO))
-                ret = _gf_false;
+	if (uuid_is_null (loc->inode->gfid)) {
+		afr_discover_do (frame, this, 0);
+		return 0;
+	}
 
-        return ret;
-}
+	afr_read_subvol_get (loc->inode, this, NULL, &event,
+			     AFR_DATA_TRANSACTION);
 
-int32_t
-afr_resultant_errno_get (int32_t *children,
-                         int *child_errno, unsigned int child_count)
-{
-        int     i = 0;
-        int32_t op_errno = 0;
-        int     child = 0;
+	if (event != local->event_generation)
+		afr_inode_refresh (frame, this, loc->inode, afr_discover_do);
+	else
+		afr_discover_do (frame, this, 0);
 
-        for (i = 0; i < child_count; i++) {
-                if (children) {
-                        child = children[i];
-                        if (child == -1)
-                                break;
-                } else {
-                        child = i;
-                }
-                if (afr_error_more_important (op_errno, child_errno[child]))
-                                op_errno = child_errno[child];
-        }
-        return op_errno;
+	return 0;
+out:
+	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+	return 0;
 }
 
-static void
-afr_lookup_handle_error (afr_local_t *local, int32_t op_ret,  int32_t op_errno)
+
+int
+afr_lookup_do (call_frame_t *frame, xlator_t *this, int err)
 {
-        GF_ASSERT (local);
-        if (op_errno == ENOENT)
-                local->enoent_count++;
+	int ret = 0;
+	int i = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
 
-        if (afr_error_more_important (local->op_errno, op_errno))
-                local->op_errno = op_errno;
+	local = frame->local;
+	priv = this->private;
 
-        if (local->op_errno == ESTALE) {
-                local->op_ret = -1;
-        }
-}
+	if (err < 0) {
+		local->op_errno = -err;
+		ret = -1;
+		goto out;
+	}
 
-static void
-afr_set_root_inode_on_first_lookup (afr_local_t *local, xlator_t *this,
-                                    inode_t *inode)
-{
-        afr_private_t           *priv = NULL;
-        GF_ASSERT (inode);
+	call_count = local->call_count = AFR_COUNT (local->child_up,
+						    priv->child_count);
 
-        if (!__is_root_gfid (inode->gfid))
-                goto out;
-        if (!afr_is_fresh_lookup (&local->loc, this))
+        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
+					    &local->loc);
+        if (ret) {
+                local->op_errno = -ret;
+		ret = -1;
                 goto out;
-        priv = this->private;
-        if ((priv->first_lookup)) {
-                gf_log (this->name, GF_LOG_INFO, "added root inode");
-                priv->root_inode = inode_ref (inode);
-                priv->first_lookup = 0;
         }
-out:
-        return;
-}
 
-static void
-afr_lookup_cache_args (afr_local_t *local, int child_index, dict_t *xattr,
-                       struct iatt *buf, struct iatt *postparent)
-{
-        GF_ASSERT (child_index >= 0);
-        local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-        local->cont.lookup.postparents[child_index] = *postparent;
-        local->cont.lookup.bufs[child_index] = *buf;
-}
-
-static void
-afr_lookup_handle_first_success (afr_local_t *local, xlator_t *this,
-                                 inode_t *inode, struct iatt *buf)
-{
-        local->cont.lookup.inode      = inode_ref (inode);
-        local->cont.lookup.buf        = *buf;
-        afr_set_root_inode_on_first_lookup (local, this, inode);
-}
-
-static void
-afr_lookup_handle_success (afr_local_t *local, xlator_t *this, int32_t child_index,
-                           int32_t op_ret, int32_t op_errno, inode_t *inode,
-                           struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
-{
-        if (local->success_count == 0) {
-                if (local->op_errno != ESTALE) {
-                        local->op_ret = op_ret;
-                        local->op_errno = 0;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_lookup_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->lookup,
+                                           &local->loc, local->xattr_req);
+                        if (!--call_count)
+                                break;
                 }
-                afr_lookup_handle_first_success (local, this, inode, buf);
         }
-        afr_lookup_update_lk_counts (local, this,
-                                     child_index, xattr);
-
-        afr_lookup_cache_args (local, child_index, xattr,
-                               buf, postparent);
-        local->cont.lookup.success_children[local->success_count] = child_index;
-        local->success_count++;
+	return 0;
+out:
+	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+	return 0;
 }
 
+/*
+ * afr_lookup()
+ *
+ * The goal here is to figure out what the element getting looked up is.
+ * i.e what is the GFID, inode type and a conservative estimate of the
+ * inode attributes are.
+ *
+ * As we lookup, operations may be underway on the entry name and the
+ * inode. In lookup() we are primarily concerned only with the entry
+ * operations. If the entry is getting unlinked or renamed, we detect
+ * what operation is underway by querying for on-going transactions and
+ * pending self-healing on the entry through xdata.
+ *
+ * If the entry is a file/dir, it may need self-heal and/or in a
+ * split-brain condition. Lookup is not the place to worry about these
+ * conditions. Outcast marking will naturally handle them in the read
+ * paths.
+ *
+ * Here is a brief goal of what we are trying to achieve:
+ *
+ * - LOOKUP on all subvolumes concurrently, querying on-going transaction
+ *   and pending self-heal info from the servers.
+ *
+ * - If all servers reply the same inode type and GFID, the overall call
+ *   MUST be a success.
+ *
+ * - If inode types or GFIDs mismatch, and there IS either an on-going
+ *   transaction or pending self-heal, inspect what the nature of the
+ *   transaction or pending heal is, and select the appropriate subvolume's
+ *   reply as the winner.
+ *
+ * - If inode types or GFIDs mismatch, and there are no on-going transactions
+ *   or pending self-heal on the entry name on any of the servers, fail the
+ *   lookup with EIO. Something has gone wrong beyond reasonable action.
+ */
+
 int
-afr_lookup_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this,  int32_t op_ret,  int32_t op_errno,
-                inode_t *inode,   struct iatt *buf, dict_t *xattr,
-                struct iatt *postparent)
+afr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-
-         child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
-
-                if (op_ret == -1) {
-                        afr_lookup_handle_error (local, op_ret, op_errno);
-                        goto unlock;
-                }
-                afr_lookup_handle_success (local, this, child_index, op_ret,
-                                           op_errno, inode, buf, xattr,
-                                           postparent);
-
-         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-               afr_lookup_done (frame, this);
+        afr_local_t   *local = NULL;
+        int32_t        op_errno = 0;
+	int            event = 0;
+
+	if (!loc->parent) {
+		afr_discover (frame, this, loc, xattr_req);
+		return 0;
+	}
+
+	if (__is_root_gfid (loc->parent->gfid)) {
+		if (!strcmp (loc->name, GF_REPLICATE_TRASH_DIR)) {
+			op_errno = EPERM;
+			goto out;
+		}
+	}
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+        if (!local->call_count) {
+                op_errno = ENOTCONN;
+                goto out;
         }
 
-         return 0;
-}
-
-int
-afr_lookup_cont_init (afr_local_t *local, unsigned int child_count)
-{
-        int               ret            = -ENOMEM;
-        struct iatt       *iatts         = NULL;
-        int32_t           *success_children = NULL;
-        int32_t           *sources       = NULL;
-
-        GF_ASSERT (local);
-        local->cont.lookup.xattrs = GF_CALLOC (child_count,
-                                               sizeof (*local->cont.lookup.xattr),
-                                               gf_afr_mt_dict_t);
-        if (NULL == local->cont.lookup.xattrs)
-                goto out;
+        local->op = GF_FOP_LOOKUP;
 
-        iatts = GF_CALLOC (child_count, sizeof (*iatts), gf_afr_mt_iatt);
-        if (NULL == iatts)
-                goto out;
-        local->cont.lookup.postparents = iatts;
+        loc_copy (&local->loc, loc);
 
-        iatts = GF_CALLOC (child_count, sizeof (*iatts), gf_afr_mt_iatt);
-        if (NULL == iatts)
-                goto out;
-        local->cont.lookup.bufs = iatts;
+	local->inode = inode_ref (loc->inode);
 
-        success_children = afr_children_create (child_count);
-        if (NULL == success_children)
-                goto out;
-        local->cont.lookup.success_children = success_children;
+	if (xattr_req)
+		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+		   allocate one for us */
+		local->xattr_req = dict_ref (xattr_req);
 
-        local->fresh_children = afr_children_create (child_count);
-        if (NULL == local->fresh_children)
-                goto out;
+	afr_read_subvol_get (loc->parent, this, NULL, &event,
+			     AFR_DATA_TRANSACTION);
 
-        sources = GF_CALLOC (sizeof (*sources), child_count, gf_afr_mt_int32_t);
-        if (NULL == sources)
-                goto out;
-        local->cont.lookup.sources = sources;
+	if (event != local->event_generation)
+		afr_inode_refresh (frame, this, loc->parent, afr_lookup_do);
+	else
+		afr_lookup_do (frame, this, 0);
 
-        ret = 0;
+	return 0;
 out:
-        return ret;
-}
-
-int
-afr_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
-{
-        afr_private_t    *priv           = NULL;
-        afr_local_t      *local          = NULL;
-        void              *gfid_req      = NULL;
-        int               ret            = -1;
-        int               i              = 0;
-        int               call_count     = 0;
-        uint64_t          ctx            = 0;
-        int32_t           op_errno       = 0;
-
-        priv = this->private;
+	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-
-        local->op_ret = -1;
+        return 0;
+}
 
-        frame->local = local;
-        local->fop = GF_FOP_LOOKUP;
 
-        if (!strcmp (loc->path, "/" GF_REPLICATE_TRASH_DIR)) {
-                op_errno = ENOENT;
-                goto out;
-        }
+/* {{{ open */
 
-        loc_copy (&local->loc, loc);
+afr_fd_ctx_t *
+__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+{
+        uint64_t       ctx = 0;
+        int            ret = 0;
+        afr_fd_ctx_t  *fd_ctx = NULL;
 
-        ret = inode_ctx_get (loc->inode, this, &ctx);
-        if (ret == 0) {
-                /* lookup is a revalidate */
+        ret = __fd_ctx_get (fd, this, &ctx);
 
-                local->read_child_index = afr_inode_get_read_ctx (this,
-                                                                  loc->inode,
-                                                                  NULL);
-        } else {
-                LOCK (&priv->read_child_lock);
-                {
-                        local->read_child_index = (++priv->read_child_rr)
-                                % (priv->child_count);
-                }
-                UNLOCK (&priv->read_child_lock);
-        }
+        if (ret < 0) {
+                ret = __afr_fd_ctx_set (this, fd);
+                if (ret < 0)
+                        goto out;
 
-        local->child_up = memdup (priv->child_up,
-                                  sizeof (*local->child_up) * priv->child_count);
-        if (NULL == local->child_up) {
-                op_errno = ENOMEM;
-                goto out;
+                ret = __fd_ctx_get (fd, this, &ctx);
+                if (ret < 0)
+                        goto out;
         }
 
-        ret = afr_lookup_cont_init (local, priv->child_count);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+out:
+        return fd_ctx;
+}
 
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        call_count = local->call_count;
-        if (local->call_count == 0) {
-                ret      = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
 
-        /* By default assume ENOTCONN. On success it will be set to 0. */
-        local->op_errno = ENOTCONN;
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+{
+        afr_fd_ctx_t  *fd_ctx = NULL;
 
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, loc,
-                                            &gfid_req);
-        if (ret) {
-                local->op_errno = -ret;
-                goto out;
-        }
-        afr_lookup_save_gfid (local->cont.lookup.gfid_req, gfid_req,
-                              loc);
-        local->fop = GF_FOP_LOOKUP;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_lookup_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
+        LOCK(&fd->lock);
+        {
+                fd_ctx = __afr_fd_ctx_get (fd, this);
         }
+        UNLOCK(&fd->lock);
 
-        ret = 0;
-out:
-        if (ret)
-                AFR_STACK_UNWIND (lookup, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
-
-        return 0;
+        return fd_ctx;
 }
 
 
-/* {{{ open */
-
 int
 __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
 {
@@ -2088,6 +1924,7 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
         int             ret    = -1;
         uint64_t        ctx    = 0;
         afr_fd_ctx_t *  fd_ctx = NULL;
+	int             i = 0;
 
         VALIDATE_OR_GOTO (this->private, out);
         VALIDATE_OR_GOTO (fd, out);
@@ -2106,21 +1943,15 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 goto out;
         }
 
-        fd_ctx->pre_op_done = GF_CALLOC (sizeof (*fd_ctx->pre_op_done),
-                                         priv->child_count,
-                                         gf_afr_mt_char);
-        if (!fd_ctx->pre_op_done) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        fd_ctx->pre_op_piggyback = GF_CALLOC (sizeof (*fd_ctx->pre_op_piggyback),
-                                              priv->child_count,
-                                              gf_afr_mt_char);
-        if (!fd_ctx->pre_op_piggyback) {
-                ret = -ENOMEM;
-                goto out;
-        }
+	for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+		fd_ctx->pre_op_done[i] = GF_CALLOC (sizeof (*fd_ctx->pre_op_done[i]),
+						    priv->child_count,
+						    gf_afr_mt_int32_t);
+		if (!fd_ctx->pre_op_done[i]) {
+			ret = -ENOMEM;
+			goto out;
+		}
+	}
 
         fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
                                        priv->child_count,
@@ -2130,6 +1961,13 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 goto out;
         }
 
+	for (i = 0; i < priv->child_count; i++) {
+		if (fd_is_anonymous (fd))
+			fd_ctx->opened_on[i] = AFR_FD_OPENED;
+		else
+			fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
+	}
+
         fd_ctx->lock_piggyback = GF_CALLOC (sizeof (*fd_ctx->lock_piggyback),
                                             priv->child_count,
                                             gf_afr_mt_char);
@@ -2146,19 +1984,9 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 goto out;
         }
 
-        fd_ctx->up_count   = priv->up_count;
-        fd_ctx->down_count = priv->down_count;
-
-        fd_ctx->locked_on = GF_CALLOC (sizeof (*fd_ctx->locked_on),
-                                       priv->child_count,
-                                       gf_afr_mt_char);
-        if (!fd_ctx->locked_on) {
-                ret = -ENOMEM;
-                goto out;
-        }
+	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
 
-        INIT_LIST_HEAD (&fd_ctx->paused_calls);
-        INIT_LIST_HEAD (&fd_ctx->entries);
+        INIT_LIST_HEAD (&fd_ctx->eager_locked);
 
         ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
         if (ret)
@@ -2186,185 +2014,90 @@ afr_fd_ctx_set (xlator_t *this, fd_t *fd)
 /* {{{ flush */
 
 int
-afr_flush_unwind (call_frame_t *frame, xlator_t *this)
+afr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (flush, main_frame,
-                                  local->op_ret, local->op_errno);
-        }
-
-        return 0;
-}
-
-
-int
-afr_flush_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-        int need_unwind = 0;
+        afr_local_t *local = NULL;
+        int call_count = -1;
 
         local = frame->local;
-        priv  = this->private;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
+			local->op_ret = op_ret;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		} else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                afr_flush_unwind (frame, this);
+	call_count = afr_frame_return (frame);
 
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	if (call_count == 0)
+		AFR_STACK_UNWIND (flush, frame, local->op_ret,
+				  local->op_errno, local->xdata_rsp);
 
         return 0;
 }
 
-
-int
-afr_flush_wind (call_frame_t *frame, xlator_t *this)
+static int
+afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int call_count = -1;
+        int           i      = 0;
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int call_count       = -1;
 
-        local = frame->local;
         priv = this->private;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
+        local = frame->local;
+        call_count = local->call_count;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_wind_cbk,
+                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->flush,
-                                           local->fd);
-
+                                           local->fd, xdata);
                         if (!--call_count)
                                 break;
+
                 }
         }
 
         return 0;
 }
 
-
 int
-afr_flush_done (call_frame_t *frame, xlator_t *this)
+afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        afr_private_t *priv  = NULL;
         afr_local_t   *local = NULL;
-        call_frame_t  *transaction_frame = NULL;
-        int            ret        = -1;
-        int            op_errno   = 0;
+        call_stub_t   *stub = NULL;
+        int            op_errno   = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        priv = this->private;
+	if (!local->call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+	local->fd = fd_ref(fd);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->op = GF_FOP_FLUSH;
-
-        local->transaction.fop    = afr_flush_wind;
-        local->transaction.done   = afr_flush_done;
-        local->transaction.unwind = afr_flush_unwind;
-
-        local->fd                 = fd_ref (fd);
-
-        local->transaction.main_frame = frame;
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
-
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
+        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
+        if (!stub)
                 goto out;
-        }
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
 
+        afr_delayed_changelog_wake_resume (this, fd, stub);
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (flush, frame, -1, op_errno);
-        }
-
+	AFR_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -2377,8 +2110,7 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         uint64_t        ctx = 0;
         afr_fd_ctx_t    *fd_ctx = NULL;
         int             ret = 0;
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
+	int             i = 0;
 
         ret = fd_ctx_get (fd, this, &ctx);
         if (ret < 0)
@@ -2387,28 +2119,22 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         fd_ctx = (afr_fd_ctx_t *)(long) ctx;
 
         if (fd_ctx) {
-                if (fd_ctx->pre_op_done)
-                        GF_FREE (fd_ctx->pre_op_done);
+                //no need to take any locks
+                if (!list_empty (&fd_ctx->eager_locked))
+                        gf_log (this->name, GF_LOG_WARNING, "%s: Stale "
+                                "Eager-lock stubs found",
+                                uuid_utoa (fd->inode->gfid));
 
-                if (fd_ctx->opened_on)
-                        GF_FREE (fd_ctx->opened_on);
+		for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++)
+			GF_FREE (fd_ctx->pre_op_done[i]);
 
-                if (fd_ctx->locked_on)
-                        GF_FREE (fd_ctx->locked_on);
+                GF_FREE (fd_ctx->opened_on);
 
-                if (fd_ctx->pre_op_piggyback)
-                        GF_FREE (fd_ctx->pre_op_piggyback);
-                list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                          call_list) {
-                        list_del_init (&paused_call->call_list);
-                        GF_FREE (paused_call);
-                }
+                GF_FREE (fd_ctx->lock_piggyback);
 
-                if (fd_ctx->lock_piggyback)
-                        GF_FREE (fd_ctx->lock_piggyback);
+                GF_FREE (fd_ctx->lock_acquired);
 
-                if (fd_ctx->lock_acquired)
-                        GF_FREE (fd_ctx->lock_acquired);
+		pthread_mutex_destroy (&fd_ctx->delay_lock);
 
                 GF_FREE (fd_ctx);
         }
@@ -2421,24 +2147,8 @@ out:
 int
 afr_release (xlator_t *this, fd_t *fd)
 {
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_locked_fd_t *tmp       = NULL;
-        afr_private_t   *priv      = NULL;
-
-        priv = this->private;
-
         afr_cleanup_fd_ctx (this, fd);
 
-        list_for_each_entry_safe (locked_fd, tmp, &priv->saved_fds,
-                                  list) {
-
-                if (locked_fd->fd == fd) {
-                        list_del_init (&locked_fd->list);
-                        GF_FREE (locked_fd);
-                }
-
-        }
-
         return 0;
 }
 
@@ -2446,51 +2156,87 @@ afr_release (xlator_t *this, fd_t *fd)
 /* {{{ fsync */
 
 int
+afr_fsync_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+        return 0;
+}
+
+int
 afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
         int child_index = (long) cookie;
-        int read_child  = 0;
+	int read_subvol = 0;
+	call_stub_t *stub = NULL;
 
         local = frame->local;
 
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+	read_subvol = afr_data_subvol_get (local->inode, this, 0, 0);
 
         LOCK (&frame->lock);
         {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
                 if (op_ret == 0) {
-                        local->op_ret = 0;
+                        if (local->op_ret == -1) {
+				local->op_ret = 0;
 
-                        if (local->success_count == 0) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
-                        }
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
 
-                        if (child_index == read_child) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+				if (xdata)
+					local->xdata_rsp = dict_ref (xdata);
                         }
 
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                        if (child_index == read_subvol) {
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
+				if (xdata) {
+					if (local->xdata_rsp)
+						dict_unref (local->xdata_rsp);
+					local->xdata_rsp = dict_ref (xdata);
+				}
+                        }
+                } else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                AFR_STACK_UNWIND (fsync, frame, local->op_ret, local->op_errno,
-                                  &local->cont.fsync.prebuf,
-                                  &local->cont.fsync.postbuf);
+		/* Make a stub out of the frame, and register it
+		   with the waking up post-op. When the call-stub resumes,
+		   we are guaranteed that there was no post-op pending
+		   (i.e changelogs were unset in the server). This is an
+		   essential "guarantee", that fsync() returns only after
+		   completely finishing EVERYTHING, including the delayed
+		   post-op. This guarantee is expected by FUSE graph switching
+		   for example.
+		*/
+		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
+                                           local->op_ret, local->op_errno,
+                                           &local->cont.inode_wfop.prebuf,
+                                           &local->cont.inode_wfop.postbuf,
+                                           local->xdata_rsp);
+		if (!stub) {
+			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+			return 0;
+		}
+
+		/* If no new unstable writes happened between the
+		   time we cleared the unstable write witness flag in afr_fsync
+		   and now, calling afr_delayed_changelog_wake_up() should
+		   wake up and skip over the fsync phase and go straight to
+		   afr_changelog_post_op_now()
+		*/
+		afr_delayed_changelog_wake_resume (this, local->fd, stub);
         }
 
         return 0;
@@ -2498,32 +2244,34 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           int32_t datasync)
+afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+	   dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
+	afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
+        int32_t op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	priv = this->private;
 
-        priv = this->private;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+        call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        local->fd = fd_ref (fd);
 
-        call_count = local->call_count;
+	if (afr_fd_has_witnessed_unstable_write (this, fd)) {
+		/* don't care. we only wanted to CLEAR the bit */
+	}
 
-        local->fd             = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -2531,16 +2279,16 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->fsync,
-                                           fd, datasync);
+                                           fd, datasync, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+	AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
         return 0;
 }
 
@@ -2548,9 +2296,9 @@ out:
 
 /* {{{ fsync */
 
-int32_t
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+int
+afr_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2559,10 +2307,13 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
                         local->op_ret = 0;
-
-                local->op_errno = op_errno;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		} else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
@@ -2570,53 +2321,49 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno);
+				  local->op_errno, local->xdata_rsp);
 
         return 0;
 }
 
 
-int32_t
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t datasync)
+int
+afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+	      dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
+	afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
-        priv = this->private;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+	priv = this->private;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_fsyncdir_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fsyncdir,
-                                    fd, datasync);
+                                    fd, datasync, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+	AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
@@ -2627,7 +2374,7 @@ out:
 int32_t
 afr_xattrop_cbk (call_frame_t *frame, void *cookie,
                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr)
+                 dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2636,8 +2383,15 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
+                        if (!local->cont.xattrop.xattr)
+                                local->cont.xattrop.xattr = dict_ref (xattr);
+
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+
                         local->op_ret = 0;
+                }
 
                 local->op_errno = op_errno;
         }
@@ -2647,7 +2401,7 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                                  xattr);
+                local->cont.xattrop.xattr, local->xdata_rsp);
 
         return 0;
 }
@@ -2655,45 +2409,41 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr)
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_xattrop_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr);
+                                    loc, optype, xattr, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -2704,7 +2454,7 @@ out:
 int32_t
 afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr)
+                  dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
 
@@ -2714,8 +2464,14 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
+                        if (!local->cont.fxattrop.xattr)
+                                local->cont.fxattrop.xattr = dict_ref (xattr);
+
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                         local->op_ret = 0;
+                }
 
                 local->op_errno = op_errno;
         }
@@ -2725,7 +2481,7 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  xattr);
+                                  local->cont.fxattrop.xattr, local->xdata_rsp);
 
         return 0;
 }
@@ -2733,45 +2489,41 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr)
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
         int32_t op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
                 goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_fxattrop_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr);
+                                    fd, optype, xattr, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -2779,8 +2531,8 @@ out:
 
 
 int32_t
-afr_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+afr_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -2801,7 +2553,7 @@ afr_inodelk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -2809,53 +2561,50 @@ afr_inodelk_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+             const char *volume, loc_t *loc, int32_t cmd,
+             struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+        local = AFR_FRAME_INIT (frame, op_errno);
+        if (!local)
                 goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_inodelk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->inodelk,
-                                    volume, loc, cmd, flock);
+                                    volume, loc, cmd, flock, xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (inodelk, frame, -1, op_errno);
+	AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 int32_t
-afr_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+afr_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -2876,62 +2625,57 @@ afr_finodelk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
 int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+afr_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+	      int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_finodelk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->finodelk,
-                                    volume, fd, cmd, flock);
+                                    volume, fd, cmd, flock, xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (finodelk, frame, -1, op_errno);
+	AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2951,63 +2695,59 @@ afr_entrylk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc,
-             const char *basename, entrylk_cmd cmd, entrylk_type type)
+int
+afr_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+	     loc_t *loc, const char *basename, entrylk_cmd cmd,
+	     entrylk_type type, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
         int32_t op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_entrylk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->entrylk,
-                                    volume, loc, basename, cmd, type);
+                                    volume, loc, basename, cmd, type, xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (entrylk, frame, -1, op_errno);
+	AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 
-int32_t
-afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+int
+afr_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -3028,148 +2768,148 @@ afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd, entrylk_type type)
+int
+afr_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+              const char *basename, entrylk_cmd cmd, entrylk_type type,
+	      dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_fentrylk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fentrylk,
-                                    volume, fd, basename, cmd, type);
+                                    volume, fd, basename, cmd, type, xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno);
+	AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
-int32_t
-afr_statfs_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno,
-                struct statvfs *statvfs)
+
+int
+afr_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+		int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = 0;
+	struct statvfs *buf = NULL;
 
         LOCK (&frame->lock);
         {
                 local = frame->local;
 
-                if (op_ret == 0) {
-                        local->op_ret   = op_ret;
-
-                        if (local->cont.statfs.buf_set) {
-                                if (statvfs->f_bavail < local->cont.statfs.buf.f_bavail)
-                                        local->cont.statfs.buf = *statvfs;
-                        } else {
-                                local->cont.statfs.buf = *statvfs;
-                                local->cont.statfs.buf_set = 1;
-                        }
-                }
-
-                if (op_ret == -1)
+                if (op_ret != 0) {
                         local->op_errno = op_errno;
-
+			goto unlock;
+		}
+
+		local->op_ret = op_ret;
+
+		buf = &local->cont.statfs.buf;
+		if (local->cont.statfs.buf_set) {
+			if (statvfs->f_bavail < buf->f_bavail) {
+				*buf = *statvfs;
+				if (xdata) {
+					if (local->xdata_rsp)
+						dict_unref (local->xdata_rsp);
+					local->xdata_rsp = dict_ref (xdata);
+				}
+			}
+		} else {
+			*buf = *statvfs;
+			local->cont.statfs.buf_set = 1;
+			if (xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		}
         }
+unlock:
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf);
+                                  &local->cont.statfs.buf, local->xdata_rsp);
 
         return 0;
 }
 
 
-int32_t
-afr_statfs (call_frame_t *frame, xlator_t *this,
-            loc_t *loc)
+int
+afr_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        afr_private_t *  priv        = NULL;
-        int              child_count = 0;
         afr_local_t   *  local       = NULL;
+	afr_private_t   *priv        = NULL;
         int              i           = 0;
-        int              ret = -1;
         int              call_count = 0;
-        int32_t          op_errno    = 0;
-
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+        int32_t          op_errno    = ENOMEM;
 
-        priv = this->private;
-        child_count = priv->child_count;
+	priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        for (i = 0; i < child_count; i++) {
+        for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_statfs_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->statfs,
-                                    loc);
+                                    loc, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
 
 int32_t
 afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                   dict_t *xdata)
 {
         afr_local_t * local = NULL;
         int call_count = -1;
@@ -3179,7 +2919,7 @@ afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  lock);
+                                  lock, xdata);
 
         return 0;
 }
@@ -3201,7 +2941,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 
         if (call_count == 0) {
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
                 return 0;
         }
 
@@ -3215,7 +2955,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
                                     priv->children[i],
                                     priv->children[i]->fops->lk,
                                     local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock);
+                                    &local->cont.lk.user_flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -3228,7 +2968,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 
 int32_t
 afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+            int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
@@ -3263,30 +3003,15 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                    priv->children[child_index],
                                    priv->children[child_index]->fops->lk,
                                    local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock);
+                                   &local->cont.lk.user_flock, xdata);
         } else if (local->op_ret == -1) {
                 /* all nodes have gone down */
 
                 AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
         } else {
-                /* locking has succeeded on all nodes that are up */
-
-                /* temporarily
-                   ret = afr_mark_locked_nodes (this, local->fd,
-                   local->cont.lk.locked_nodes);
-                   if (ret)
-                   gf_log (this->name, GF_LOG_DEBUG,
-                   "Could not save locked nodes info in fdctx");
-
-                   ret = afr_save_locked_fd (this, local->fd);
-                   if (ret)
-                   gf_log (this->name, GF_LOG_DEBUG,
-                   "Could not save locked fd");
-
-                */
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
         }
 
         return 0;
@@ -3295,25 +3020,17 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd, struct gf_flock *flock)
+        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
         int i = 0;
-        int32_t op_errno = 0;
-        int     ret      = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+        local = AFR_FRAME_INIT (frame, op_errno);
+        if (!local)
                 goto out;
 
         local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
@@ -3333,31 +3050,18 @@ afr_lk (call_frame_t *frame, xlator_t *this,
         STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
                            priv->children[i],
                            priv->children[i]->fops->lk,
-                           fd, cmd, flock);
+                           fd, cmd, flock, xdata);
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
 int
 afr_forget (xlator_t *this, inode_t *inode)
 {
-        uint64_t        ctx_addr = 0;
-        afr_inode_ctx_t *ctx     = NULL;
-
-        inode_ctx_get (inode, this, &ctx_addr);
-
-        if (!ctx_addr)
-                goto out;
-
-        ctx = (afr_inode_ctx_t *)(long)ctx_addr;
-        if (ctx->fresh_children)
-                GF_FREE (ctx->fresh_children);
-        GF_FREE (ctx);
-out:
         return 0;
 }
 
@@ -3377,7 +3081,6 @@ afr_priv_dump (xlator_t *this)
         snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
         gf_proc_dump_add_section(key_prefix);
         gf_proc_dump_write("child_count", "%u", priv->child_count);
-        gf_proc_dump_write("read_child_rr", "%u", priv->read_child_rr);
         for (i = 0; i < priv->child_count; i++) {
                 sprintf (key, "child_up[%d]", i);
                 gf_proc_dump_write(key, "%d", priv->child_up[i]);
@@ -3434,7 +3137,7 @@ afr_notify (xlator_t *this, int32_t event,
         int             idx                 = -1;
         int             ret                 = -1;
         int             call_psh            = 0;
-        int             up_child            = AFR_ALL_CHILDREN;
+        int             up_child            = -1;
         dict_t          *input              = NULL;
         dict_t          *output             = NULL;
 
@@ -3443,6 +3146,14 @@ afr_notify (xlator_t *this, int32_t event,
         if (!priv)
                 return 0;
 
+        /*
+         * We need to reset this in case children come up in "staggered"
+         * fashion, so that we discover a late-arriving local subvolume.  Note
+         * that we could end up issuing N lookups to the first subvolume, and
+         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+         */
+        priv->did_discovery = _gf_false;
+
         had_heard_from_all = 1;
         for (i = 0; i < priv->child_count; i++) {
                 if (!priv->last_event[i]) {
@@ -3470,8 +3181,17 @@ afr_notify (xlator_t *this, int32_t event,
         case GF_EVENT_CHILD_UP:
                 LOCK (&priv->lock);
                 {
+                        /*
+                         * This only really counts if the child was never up
+                         * (value = -1) or had been down (value = 0).  See
+                         * comment at GF_EVENT_CHILD_DOWN for a more detailed
+                         * explanation.
+                         */
+                        if (priv->child_up[idx] != 1) {
+                                priv->up_count++;
+				priv->event_generation++;
+                        }
                         priv->child_up[idx] = 1;
-                        priv->up_count++;
 
                         call_psh = 1;
                         up_child = idx;
@@ -3495,8 +3215,23 @@ afr_notify (xlator_t *this, int32_t event,
         case GF_EVENT_CHILD_DOWN:
                 LOCK (&priv->lock);
                 {
+                        /*
+                         * If a brick is down when we start, we'll get a
+                         * CHILD_DOWN to indicate its initial state.  There
+                         * was never a CHILD_UP in this case, so if we
+                         * increment "down_count" the difference between than
+                         * and "up_count" will no longer be the number of
+                         * children that are currently up.  This has serious
+                         * implications e.g. for quorum enforcement, so we
+                         * don't increment these values unless the event
+                         * represents an actual state transition between "up"
+                         * (value = 1) and anything else.
+                         */
+                        if (priv->child_up[idx] == 1) {
+                                priv->down_count++;
+				priv->event_generation++;
+                        }
                         priv->child_up[idx] = 0;
-                        priv->down_count++;
 
                         for (i = 0; i < priv->child_count; i++)
                                 if (priv->child_up[i] == 0)
@@ -3527,6 +3262,10 @@ afr_notify (xlator_t *this, int32_t event,
         case GF_EVENT_TRANSLATOR_OP:
                 input = data;
                 output = data2;
+                if (!had_heard_from_all) {
+                        ret = -1;
+                        goto out;
+                }
                 ret = afr_xl_op (this, input, output);
                 goto out;
                 break;
@@ -3557,8 +3296,7 @@ afr_notify (xlator_t *this, int32_t event,
 
                 LOCK (&priv->lock);
                 {
-                        up_children = afr_up_children_count (priv->child_up,
-                                                             priv->child_count);
+                        up_children = AFR_COUNT (priv->child_up, priv->child_count);
                         for (i = 0; i < priv->child_count; i++) {
                                 if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
                                         event = GF_EVENT_CHILD_UP;
@@ -3578,39 +3316,23 @@ afr_notify (xlator_t *this, int32_t event,
         ret = 0;
         if (propagate)
                 ret = default_notify (this, event, data);
-        if (call_psh && priv->shd.iamshd)
-                afr_do_poll_self_heal ((void*) (long) up_child);
 
+        if (call_psh && priv->shd.iamshd) {
+                afr_selfheal_childup (this, up_child);
+	}
 out:
         return ret;
 }
 
-int
-afr_first_up_child (unsigned char *child_up, size_t child_count)
-{
-        int         ret      = -1;
-        int         i        = 0;
-
-        GF_ASSERT (child_up);
-
-        for (i = 0; i < child_count; i++) {
-                if (child_up[i]) {
-                        ret = i;
-                        break;
-                }
-        }
-
-        return ret;
-}
 
 int
 afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
 {
-        int     ret = -1;
-
         local->op_ret = -1;
         local->op_errno = EUCLEAN;
 
+	syncbarrier_init (&local->barrier);
+
         local->child_up = GF_CALLOC (priv->child_count,
                                      sizeof (*local->child_up),
                                      gf_afr_mt_char);
@@ -3622,27 +3344,42 @@ afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
 
         memcpy (local->child_up, priv->child_up,
                 sizeof (*local->child_up) * priv->child_count);
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
+        local->call_count = AFR_COUNT (local->child_up, priv->child_count);
         if (local->call_count == 0) {
                 gf_log (THIS->name, GF_LOG_INFO, "no subvolumes up");
                 if (op_errno)
                         *op_errno = ENOTCONN;
                 goto out;
         }
-
-        local->child_errno = GF_CALLOC (priv->child_count,
-                                        sizeof (*local->child_errno),
-                                        gf_afr_mt_int32_t);
-        if (!local->child_errno) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
-
-        ret = 0;
+	local->event_generation = priv->event_generation;
+
+	local->read_attempted = GF_CALLOC (priv->child_count, sizeof (char),
+					   gf_afr_mt_char);
+	if (!local->read_attempted) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
+
+	local->readable = GF_CALLOC (priv->child_count, sizeof (char),
+				     gf_afr_mt_char);
+	if (!local->readable) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
+
+	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+				   gf_afr_mt_reply_t);
+	if (!local->replies) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
+
+	return 0;
 out:
-        return ret;
+        return -1;
 }
 
 int
@@ -3651,16 +3388,6 @@ afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
 {
         int             ret = -ENOMEM;
 
-        lk->inode_locked_nodes = GF_CALLOC (sizeof (*lk->inode_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->inode_locked_nodes)
-                goto out;
-
-        lk->entry_locked_nodes = GF_CALLOC (sizeof (*lk->entry_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->entry_locked_nodes)
-                goto out;
-
         lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
                                       child_count, gf_afr_mt_char);
         if (NULL == lk->locked_nodes)
@@ -3719,6 +3446,21 @@ out:
 }
 
 int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count)
+{
+        int             ret = -ENOMEM;
+
+        lk->domain = dom;
+        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
+                                      child_count, gf_afr_mt_char);
+        if (NULL == lk->locked_nodes)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 afr_transaction_local_init (afr_local_t *local, xlator_t *this)
 {
         int            child_up_count = 0;
@@ -3731,14 +3473,20 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (ret < 0)
                 goto out;
 
+        if ((local->transaction.type == AFR_DATA_TRANSACTION) ||
+            (local->transaction.type == AFR_METADATA_TRANSACTION)) {
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[0],
+                                        this->name, priv->child_count);
+                if (ret < 0)
+                        goto out;
+        }
+
         ret = -ENOMEM;
-        child_up_count = afr_up_children_count (local->child_up,
-                                                priv->child_count);
+        child_up_count = AFR_COUNT (local->child_up, priv->child_count);
         if (priv->optimistic_change_log && child_up_count == priv->child_count)
                 local->optimistic_change_log = 1;
 
-        local->first_up_child = afr_first_up_child (local->child_up,
-                                                    priv->child_count);
+	local->pre_op_compat = priv->pre_op_compat;
 
         local->transaction.eager_lock =
                 GF_CALLOC (sizeof (*local->transaction.eager_lock),
@@ -3748,120 +3496,36 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (!local->transaction.eager_lock)
                 goto out;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children)
-                goto out;
-
-        if (local->fd) {
-                local->fd_open_on = GF_CALLOC (sizeof (*local->fd_open_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!local->fd_open_on)
-                        goto out;
-        }
-
         local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
                                                priv->child_count,
                                                gf_afr_mt_char);
         if (!local->transaction.pre_op)
                 goto out;
 
+        local->transaction.fop_subvols = GF_CALLOC (sizeof (*local->transaction.fop_subvols),
+						    priv->child_count,
+						    gf_afr_mt_char);
+        if (!local->transaction.fop_subvols)
+                goto out;
+
+        local->transaction.failed_subvols = GF_CALLOC (sizeof (*local->transaction.failed_subvols),
+						       priv->child_count,
+						       gf_afr_mt_char);
+        if (!local->transaction.failed_subvols)
+                goto out;
+
         local->pending = afr_matrix_create (priv->child_count,
                                             AFR_NUM_CHANGE_LOGS);
         if (!local->pending)
                 goto out;
 
-        local->transaction.child_errno =
-                GF_CALLOC (sizeof (*local->transaction.child_errno),
-                           priv->child_count,
-                           gf_afr_mt_int32_t);
-        local->transaction.erase_pending = 1;
+	INIT_LIST_HEAD (&local->transaction.eager_locked);
 
         ret = 0;
 out:
         return ret;
 }
 
-void
-afr_reset_children (int32_t *fresh_children, int32_t child_count)
-{
-        unsigned int i = 0;
-        for (i = 0; i < child_count; i++)
-                fresh_children[i] = -1;
-}
-
-int32_t*
-afr_children_create (int32_t child_count)
-{
-        int32_t           *children = NULL;
-        int               i               = 0;
-
-        GF_ASSERT (child_count > 0);
-
-        children = GF_CALLOC (child_count, sizeof (*children),
-                              gf_afr_mt_int32_t);
-        if (NULL == children)
-                goto out;
-        for (i = 0; i < child_count; i++)
-                children[i] = -1;
-out:
-        return children;
-}
-
-void
-afr_children_add_child (int32_t *children, int32_t child,
-                        int32_t child_count)
-{
-        gf_boolean_t child_found = _gf_false;
-        int          i               = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                if (children[i] == child) {
-                        child_found = _gf_true;
-                        break;
-                }
-        }
-
-        if (!child_found) {
-                GF_ASSERT (i < child_count);
-                children[i] = child;
-        }
-}
-
-void
-afr_children_rm_child (int32_t *children, int32_t child, int32_t child_count)
-{
-        int          i = 0;
-
-        GF_ASSERT ((child >= 0) && (child < child_count));
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                if (children[i] == child) {
-                        if (i != (child_count - 1))
-                                memmove (children + i, children + i + 1,
-                                         sizeof (*children)*(child_count - i - 1));
-                        children[child_count - 1] = -1;
-                        break;
-                }
-        }
-}
-
-int
-afr_get_children_count (int32_t *children, unsigned int child_count)
-{
-        int count = 0;
-        int i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                count++;
-        }
-        return count;
-}
 
 void
 afr_set_low_priority (call_frame_t *frame)
@@ -3869,39 +3533,6 @@ afr_set_low_priority (call_frame_t *frame)
         frame->root->pid = LOW_PRIO_PROC_PID;
 }
 
-int
-afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags, int32_t wbflags)
-{
-        int             ret = 0;
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx      = NULL;
-
-        GF_ASSERT (fd && fd->inode);
-        ret = afr_fd_ctx_set (this, fd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not set fd ctx for fd=%p", fd);
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not get fd ctx for fd=%p", fd);
-                goto out;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-        fd_ctx->opened_on[child] = AFR_FD_OPENED;
-        if (!IA_ISDIR (fd->inode->ia_type)) {
-                fd_ctx->flags            = flags;
-                fd_ctx->wbflags          = wbflags;
-        }
-        ret = 0;
-out:
-        return ret;
-}
 
 gf_boolean_t
 afr_have_quorum (char *logname, afr_private_t *priv)
@@ -3948,24 +3579,6 @@ afr_priv_destroy (afr_private_t *priv)
         if (!priv)
                 goto out;
         inode_unref (priv->root_inode);
-        GF_FREE (priv->shd.pos);
-        GF_FREE (priv->shd.pending);
-        GF_FREE (priv->shd.inprogress);
-        GF_FREE (priv->shd.sh_times);
-//        for (i = 0; i < priv->child_count; i++)
-//                if (priv->shd.timer && priv->shd.timer[i])
-//                        gf_timer_call_cancel (this->ctx, priv->shd.timer[i]);
-        GF_FREE (priv->shd.timer);
-
-        if (priv->shd.healed)
-                eh_destroy (priv->shd.healed);
-
-        if (priv->shd.heal_failed)
-                eh_destroy (priv->shd.heal_failed);
-
-        if (priv->shd.split_brain)
-                eh_destroy (priv->shd.split_brain);
-
         GF_FREE (priv->last_event);
         if (priv->pending_key) {
                 for (i = 0; i < priv->child_count; i++)
@@ -3975,8 +3588,7 @@ afr_priv_destroy (afr_private_t *priv)
         GF_FREE (priv->children);
         GF_FREE (priv->child_up);
         LOCK_DESTROY (&priv->lock);
-        LOCK_DESTROY (&priv->read_child_lock);
-        pthread_mutex_destroy (&priv->mutex);
+
         GF_FREE (priv);
 out:
         return;
@@ -3992,3 +3604,22 @@ xlator_subvolume_count (xlator_t *this)
                 i++;
         return i;
 }
+
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_fd_ctx_t    *fd_ctx   = NULL;
+
+        local = frame->local;
+
+        if (!local->fd)
+		return;
+
+	fd_ctx = afr_fd_ctx_get (local->fd, this);
+	if (!fd_ctx)
+		return;
+
+	fd_ctx->open_fd_count = local->open_fd_count;
+}
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index ee9c5d8cc..fa1da3958 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -46,687 +37,374 @@
 #include "checksum.h"
 
 #include "afr.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-int
-afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                           int32_t op_errno)
-{
-        afr_local_t *local  = NULL;
-
-        local = frame->local;
-
-        afr_set_opendir_done (this, local->fd->inode);
-
-        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd);
-
-        return 0;
-}
-
-
-gf_boolean_t
-__checksums_differ (uint32_t *checksum, int child_count,
-                    unsigned char *child_up)
-{
-        int          ret            = _gf_false;
-        int          i              = 0;
-        uint32_t     cksum          = 0;
-        gf_boolean_t activate_check = _gf_false;
-
-        for (i = 0; i < child_count; i++) {
-                if (!child_up[i])
-                        continue;
-                if (_gf_false == activate_check) {
-                        cksum          = checksum[i];
-                        activate_check = _gf_true;
-                        continue;
-                }
-
-                if (cksum != checksum[i]) {
-                        ret = _gf_true;
-                        break;
-                }
-
-                cksum = checksum[i];
-        }
-
-        return ret;
-}
-
-
-int32_t
-afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             gf_dirent_t *entries)
-{
-        afr_private_t *   priv        = NULL;
-        afr_local_t *     local       = NULL;
-        afr_self_heal_t * sh          = NULL;
-        gf_dirent_t *     entry       = NULL;
-        gf_dirent_t *     tmp         = NULL;
-        char              *reason     = NULL;
-        int               child_index = 0;
-        uint32_t          entry_cksum = 0;
-        int               call_count  = 0;
-        off_t             last_offset = 0;
-        inode_t           *inode      = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        inode = local->fd->inode;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to do opendir on %s",
-                        local->loc.path, priv->children[child_index]->name);
-                local->op_ret = -1;
-                local->op_ret = op_errno;
-                goto out;
-        }
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: no entries found in %s",
-                        local->loc.path, priv->children[child_index]->name);
-                goto out;
-        }
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                entry_cksum = gf_rsync_weak_checksum (entry->d_name,
-                                                      strlen (entry->d_name));
-                local->cont.opendir.checksum[child_index] ^= entry_cksum;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-        }
-
-        /* read more entries */
-
-        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->readdir,
-                           local->fd, 131072, last_offset);
-
-        return 0;
-
-out:
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (__checksums_differ (local->cont.opendir.checksum,
-                                        priv->child_count,
-                                        local->child_up)) {
-
-                        sh->do_entry_self_heal  = _gf_true;
-                        sh->forced_merge          = _gf_true;
-
-                        reason = "checksums of directory differ";
-                        afr_launch_self_heal (frame, this, inode, _gf_false,
-                                              inode->ia_type, reason, NULL,
-                                              afr_examine_dir_sh_unwind);
-                } else {
-                        afr_set_opendir_done (this, inode);
-
-                        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                          local->op_errno, local->fd);
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_examine_dir (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv       = NULL;
-        afr_local_t *   local      = NULL;
-        int             i          = 0;
-        int             call_count = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        local->cont.opendir.checksum = GF_CALLOC (priv->child_count,
-                                                  sizeof (*local->cont.opendir.checksum),
-                                                  gf_afr_mt_int32_t);
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->readdir,
-                                           local->fd, 131072, 0);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
+#include "afr-transaction.h"
 
 
 int32_t
 afr_opendir_cbk (call_frame_t *frame, void *cookie,
                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 fd_t *fd)
+                 fd_t *fd, dict_t *xdata)
 {
-        afr_private_t *priv              = NULL;
         afr_local_t   *local             = NULL;
-        int32_t        up_children_count = 0;
-        int            ret               = -1;
         int            call_count        = -1;
         int32_t        child_index       = 0;
+	afr_fd_ctx_t  *fd_ctx = NULL;
 
-        priv  = this->private;
         local = frame->local;
+	fd_ctx = local->fd_ctx;
         child_index = (long) cookie;
 
-        up_children_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-
         LOCK (&frame->lock);
         {
-                if (op_ret >= 0) {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+                } else {
                         local->op_ret = op_ret;
-                        ret = afr_child_fd_ctx_set (this, fd, child_index,
-                                                    0, 0);
-                        if (ret) {
-                                local->op_ret = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                 }
-
-                local->op_errno = op_errno;
         }
-unlock:
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                if (local->op_ret != 0)
-                        goto out;
-
-                if (!afr_is_opendir_done (this, local->fd->inode) &&
-                    up_children_count > 1) {
-
-                        /*
-                         * This is the first opendir on this inode. We need
-                         * to check if the directory's entries are the same
-                         * on all subvolumes. This is needed in addition
-                         * to regular entry self-heal because the readdir
-                         * call is sent only to the first subvolume, and
-                         * thus files that exist only there will never be healed
-                         * otherwise (assuming changelog shows no anomalies).
-                         */
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "reading contents of directory %s looking for mismatch",
-                                local->loc.path);
-
-                        afr_examine_dir (frame, this);
-
-                } else {
-                        /* do the unwind */
-                        goto out;
-                }
-        }
-
-        return 0;
-
-out:
-        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd);
-
+        if (call_count == 0)
+		AFR_STACK_UNWIND (opendir, frame, local->op_ret,
+				  local->op_errno, local->fd, NULL);
         return 0;
 }
 
 
-int32_t
-afr_opendir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, fd_t *fd)
+int
+afr_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 {
         afr_private_t * priv        = NULL;
         afr_local_t   * local       = NULL;
-        int             child_count = 0;
         int             i           = 0;
-        int             ret         = -1;
         int             call_count  = -1;
-        int32_t         op_errno    = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t         op_errno    = ENOMEM;
+	afr_fd_ctx_t *fd_ctx = NULL;
 
         priv = this->private;
 
-        child_count = priv->child_count;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		goto out;
 
         loc_copy (&local->loc, loc);
 
         local->fd    = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
 
         call_count = local->call_count;
 
-        for (i = 0; i < child_count; i++) {
+        for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_opendir_cbk,
                                            (void*) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->opendir,
-                                           loc, fd);
+                                           loc, fd, NULL);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd);
-
+	AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
         return 0;
 }
 
 
-/**
- * Common algorithm for directory read calls:
- *
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: readdir
- */
-
+#define BACKEND_D_OFF_BITS 63
+#define PRESENT_D_OFF_BITS 63
 
-struct entry_name {
-        char *name;
-        struct list_head list;
-};
+#define ONE 1ULL
+#define MASK (~0ULL)
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
+#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS))
 
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
 
-static gf_boolean_t
-remembered_name (const char *name, struct list_head *entries)
+static uint64_t
+afr_bits_for (uint64_t num)
 {
-        struct entry_name *e   = NULL;
-        gf_boolean_t       ret = _gf_false;
+	uint64_t bits = 0, ctrl = 1;
 
-        list_for_each_entry (e, entries, list) {
-                if (!strcmp (name, e->name)) {
-                        ret = _gf_true;
-                        goto out;
-                }
-        }
+	while (ctrl < num) {
+		ctrl *= 2;
+		bits ++;
+	}
 
-out:
-        return ret;
+	return bits;
 }
 
-
-static void
-afr_remember_entries (gf_dirent_t *entries, fd_t *fd)
+int
+afr_itransform (xlator_t *this, int subvol, uint64_t x, uint64_t *y_p)
 {
-        struct entry_name *n      = NULL;
-        gf_dirent_t       *entry  = NULL;
-        int                ret    = 0;
-        uint64_t           ctx    = 0;
-        afr_fd_ctx_t      *fd_ctx = NULL;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return;
+        afr_private_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    y = 0;
+        uint64_t    hi_mask = 0;
+        uint64_t    off_mask = 0;
+        int         max_bits = 0;
+
+        if (x == ((uint64_t) -1)) {
+                y = (uint64_t) -1;
+                goto out;
         }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                n = GF_CALLOC (1, sizeof (*n), gf_afr_mt_entry_name);
-                n->name = gf_strdup (entry->d_name);
-                INIT_LIST_HEAD (&n->list);
-
-                list_add (&n->list, &fd_ctx->entries);
-        }
-}
+        conf = this->private;
+        if (!conf)
+                goto out;
 
+        max = conf->child_count;
+        cnt = subvol;
 
-static off_t
-afr_filter_entries (gf_dirent_t *entries, fd_t *fd)
-{
-        gf_dirent_t  *entry  = NULL;
-        gf_dirent_t  *tmp    = NULL;
-        int           ret    = 0;
-        uint64_t      ctx    = 0;
-        afr_fd_ctx_t *fd_ctx = NULL;
-        off_t         offset = 0;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return -1;
-        }
+	if (max == 1) {
+		y = x;
+		goto out;
+	}
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        max_bits = afr_bits_for (max);
 
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                offset = entry->d_off;
+        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
 
-                if (remembered_name (entry->d_name, &fd_ctx->entries)) {
-                        list_del (&entry->list);
-                        GF_FREE (entry);
-                }
+        if (x & hi_mask) {
+                /* HUGE d_off */
+                off_mask = MASK << max_bits;
+                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | cnt;
+        } else {
+                /* small d_off */
+                y = ((x * max) + cnt);
         }
 
-        return offset;
+out:
+        if (y_p)
+                *y_p = y;
+
+        return 0;
 }
 
 
-static void
-afr_forget_entries (fd_t *fd)
+int
+afr_deitransform (xlator_t *this, uint64_t y, int *subvol_p,
+                  uint64_t *x_p)
 {
-        struct entry_name *entry  = NULL;
-        struct entry_name *tmp    = NULL;
-        int                ret    = 0;
-        uint64_t           ctx    = 0;
-        afr_fd_ctx_t      *fd_ctx = NULL;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return;
-        }
+        afr_private_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    x = 0;
+        int         subvol = 0;
+        int         max_bits = 0;
+        uint64_t    off_mask = 0;
+        uint64_t    host_mask = 0;
+
+        if (!this->private)
+                return -1;
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        conf = this->private;
+        max = conf->child_count;
 
-        list_for_each_entry_safe (entry, tmp, &fd_ctx->entries, list) {
-                GF_FREE (entry->name);
-                list_del (&entry->list);
-                GF_FREE (entry);
-        }
-}
+	if (max == 1) {
+		x = y;
+		cnt = 0;
+		goto out;
+	}
 
+        if (y & TOP_BIT) {
+                /* HUGE d_off */
+                max_bits = afr_bits_for (max);
+                off_mask = (MASK << max_bits);
+                host_mask = ~(off_mask);
 
-int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 gf_dirent_t *entries)
-{
-        afr_local_t *   local       = NULL;
-        gf_dirent_t *   entry       = NULL;
-        gf_dirent_t *   tmp         = NULL;
+                x = ((y & ~TOP_BIT) & off_mask) << SHIFT_BITS;
 
-        local = frame->local;
-
-        if (op_ret == -1)
-                goto out;
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if ((local->fd->inode == local->fd->inode->table->root)
-                    && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-                        list_del_init (&entry->list);
-                        GF_FREE (entry);
-                }
+                cnt = y & host_mask;
+	} else {
+                /* small d_off */
+                cnt = y % max;
+                x = y / max;
         }
 
 out:
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries);
+        subvol = cnt;
+
+        if (subvol_p)
+                *subvol_p = subvol;
+
+        if (x_p)
+                *x_p = x;
 
         return 0;
 }
 
 
-int32_t
-afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+static void
+afr_readdir_transform_entries (gf_dirent_t *subvol_entries, int subvol,
+			       gf_dirent_t *entries, fd_t *fd)
 {
-        afr_private_t *  priv            = NULL;
-        afr_local_t *    local           = NULL;
-        xlator_t **      children        = NULL;
-        int32_t          next_call_child = -1;
-        int              ret             = 0;
-        gf_dirent_t *    entry           = NULL;
-        gf_dirent_t *    tmp             = NULL;
-        int32_t          *last_index     = NULL;
-        int32_t          read_child      = -1;
-        int32_t         *fresh_children   = NULL;
-        uint64_t         ctx             = 0;
-        afr_fd_ctx_t    *fd_ctx          = NULL;
-        off_t            offset          = 0;
-        int32_t         call_child       = -1;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-
-        read_child = (long) cookie;
-        last_index = &local->cont.readdir.last_index;
-        fresh_children = local->fresh_children;
-
-        /* the value of the last_index changes if afr_next_call_child is
-         * called. So to find the call_child of this callback use last_index
-         * before the next_call_child call.
-         */
-        if (*last_index == -1)
-                call_child = read_child;
-        else
-                call_child = fresh_children[*last_index];
-
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", local->fd);
-                        op_ret   = -1;
-                        op_errno = -ret;
-                        goto out;
+	afr_private_t *priv = NULL;
+        gf_dirent_t *entry = NULL;
+        gf_dirent_t *tmp = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	int gen = 0;
+
+	priv = THIS->private;
+
+	data_readable = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+
+        list_for_each_entry_safe (entry, tmp, &subvol_entries->list, list) {
+                if (__is_root_gfid (fd->inode->gfid) &&
+                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
+			continue;
                 }
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+		list_del_init (&entry->list);
+		afr_itransform (THIS, subvol, entry->d_off, &entry->d_off);
+		list_add_tail (&entry->list, &entries->list);
 
-                if (op_ret == -1) {
-                        next_call_child = afr_next_call_child (fresh_children,
-                                                               local->child_up,
-                                                               priv->child_count,
-                                                               last_index,
-                                                               read_child);
-                        if (next_call_child < 0)
-                                goto out;
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "starting readdir afresh on child %d, offset %"PRId64,
-                                next_call_child, (uint64_t) 0);
-
-                        fd_ctx->failed_over = _gf_true;
-
-                        STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                           (void *) (long) read_child,
-                                           children[next_call_child],
-                                           children[next_call_child]->fops->readdirp,
-                                           local->fd,
-                                           local->cont.readdir.size, 0,
-                                           local->cont.readdir.dict);
-                        return 0;
-                }
-        }
+		if (entry->inode) {
+			gen = 0;
+			afr_inode_read_subvol_get (entry->inode, THIS,
+						   data_readable,
+						   metadata_readable, &gen);
 
-        if (op_ret != -1) {
-                list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                        if ((local->fd->inode == local->fd->inode->table->root)
-                            && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-                                list_del_init (&entry->list);
-                                GF_FREE (entry);
-                        }
-                }
-        }
+			if (gen != priv->event_generation ||
+				!data_readable[subvol] ||
+				!metadata_readable[subvol]) {
 
-        if (priv->strict_readdir) {
-                if (fd_ctx->failed_over) {
-                        if (list_empty (&entries->list)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "no entries found");
-                                goto out;
-                        }
-
-                        offset = afr_filter_entries (entries, local->fd);
-
-                        afr_remember_entries (entries, local->fd);
-
-                        if (list_empty (&entries->list)) {
-                                /* All the entries we got were duplicate. We
-                                   shouldn't send an empty list now, because
-                                   that will make the application stop reading. So
-                                   try to get more entries */
-
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "trying to fetch non-duplicate entries "
-                                        "from offset %"PRId64", child %s",
-                                        offset, children[call_child]->name);
-
-                                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                                   (void *) (long) read_child,
-                                                   children[call_child],
-                                                   children[call_child]->fops->readdirp,
-                                                   local->fd, local->cont.readdir.size, offset,
-                                                   local->cont.readdir.dict);
-                                return 0;
-                        }
-                } else {
-                        afr_remember_entries (entries, local->fd);
-                }
+				inode_unref (entry->inode);
+				entry->inode = NULL;
+			}
+		}
         }
-
-out:
-        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries);
-
-        return 0;
 }
 
+
 int32_t
-afr_do_readdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset, int whichop, dict_t *dict)
+afr_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
+		 dict_t *xdata)
 {
-        afr_private_t *  priv       = NULL;
-        xlator_t **      children   = NULL;
-        int              call_child = 0;
-        afr_local_t     *local      = NULL;
-        uint64_t         ctx        = 0;
-        afr_fd_ctx_t    *fd_ctx     = NULL;
-        int              ret        = -1;
-        int32_t          op_errno   = 0;
-        uint64_t         read_child = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        afr_local_t *local = NULL;
+	gf_dirent_t  entries;
+
+	INIT_LIST_HEAD (&entries.list);
+
         local = frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        if (op_ret < 0 && !local->cont.readdir.offset) {
+		/* failover only if this was first readdir, detected
+		   by offset == 0 */
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readdir.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	if (op_ret >= 0)
+		afr_readdir_transform_entries (subvol_entries, (long) cookie,
+					       &entries, local->fd);
 
-        local->fd                  = fd_ref (fd);
-        local->cont.readdir.size   = size;
-        local->cont.readdir.dict   = (dict)? dict_ref (dict) : NULL;
-
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", fd);
-                        op_errno = -ret;
-                        goto out;
-                }
+        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, xdata);
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        return 0;
+}
 
-                if (fd_ctx->last_tried != call_child) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "first up child has changed from %d to %d, "
-                                "restarting readdir from offset 0",
-                                fd_ctx->last_tried, call_child);
 
-                        fd_ctx->failed_over = _gf_true;
-                        offset = 0;
-                }
+int
+afr_readdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
-                fd_ctx->last_tried = call_child;
-        }
+	priv = this->private;
+	local = frame->local;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readdir, frame, local->op_ret,
+				  local->op_errno, 0, 0);
+		return 0;
+	}
 
-        if (whichop == GF_FOP_READDIR)
+        if (local->op == GF_FOP_READDIR)
                 STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdir, fd,
-                                   size, offset);
+                                   (void *) (long) subvol,
+                                   priv->children[subvol],
+                                   priv->children[subvol]->fops->readdir,
+				   local->fd, local->cont.readdir.size,
+                                   local->cont.readdir.offset,
+				   local->xdata_req);
         else
-                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdirp, fd,
-                                   size, offset, dict);
+                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
+                                   (void *) (long) subvol,
+                                   priv->children[subvol],
+                                   priv->children[subvol]->fops->readdirp,
+				   local->fd, local->cont.readdir.size,
+				   local->cont.readdir.offset,
+				   local->xdata_req);
+	return 0;
+}
+
 
-        ret = 0;
+int
+afr_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+		off_t offset, int whichop, dict_t *dict)
+{
+        afr_local_t   *local     = NULL;
+        int32_t       op_errno   = 0;
+	int           subvol = -1;
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+	local->op = whichop;
+        local->fd = fd_ref (fd);
+        local->cont.readdir.size = size;
+	local->cont.readdir.offset = offset;
+        local->xdata_req = (dict)? dict_ref (dict) : NULL;
+
+	if (offset == 0) {
+		/* First readdir has option of failing over and selecting
+		   an appropriate read subvolume */
+		afr_read_txn (frame, this, fd->inode, afr_readdir_wind,
+			      AFR_DATA_TRANSACTION);
+	} else {
+		/* But continued readdirs MUST stick to the same subvolume
+		   without an option to failover */
+		afr_deitransform (this, offset, &subvol,
+				  (uint64_t *)&local->cont.readdir.offset);
+		afr_readdir_wind (frame, this, subvol);
+	}
+
+        return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset)
+             off_t offset, dict_t *xdata)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, NULL);
+	afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+
         return 0;
 }
 
@@ -736,6 +414,7 @@ afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
               off_t offset, dict_t *dict)
 {
         afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
+
         return 0;
 }
 
@@ -743,7 +422,6 @@ afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 int32_t
 afr_releasedir (xlator_t *this, fd_t *fd)
 {
-        afr_forget_entries (fd);
         afr_cleanup_fd_ctx (this, fd);
 
         return 0;
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 7e50a1c8c..09456d159 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_READ_H__
@@ -23,14 +14,14 @@
 
 int32_t
 afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd);
+	     loc_t *loc, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_releasedir (xlator_t *this, fd_t *fd);
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset);
+	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
 
 
 int32_t
@@ -39,7 +30,7 @@ afr_readdirp (call_frame_t *frame, xlator_t *this,
 
 int32_t
 afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags);
+	      loc_t *loc, int32_t flags, dict_t *xdata);
 
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 1cd447e75..465dde54f 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -43,10 +34,14 @@
 #include "common-utils.h"
 #include "compat-errno.h"
 #include "compat.h"
+#include "byte-order.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
 
+void
+afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this);
+
 int
 afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
 {
@@ -65,48 +60,192 @@ afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
                         *op_errno = ENOMEM;
                 goto out;
         }
-        parent->path = dirname (child_path);
-        parent->inode  = inode_ref (child->parent);
-        uuid_copy (parent->gfid, child->pargfid);
+
+        parent->path = gf_strdup (dirname (child_path));
+        if (!parent->path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+
+        parent->inode = inode_ref (child->parent);
+	uuid_copy (parent->gfid, child->pargfid);
 
         ret = 0;
 out:
+        GF_FREE (child_path);
+
         return ret;
 }
 
-/* {{{ create */
 
-int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_dir_write_finalize (call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-        struct iatt  *unwind_buf = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int inode_read_subvol = -1;
+	int parent_read_subvol = -1;
+	int parent2_read_subvol = -1;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (local->inode) {
+		afr_replies_interpret (frame, this, local->inode);
+		inode_read_subvol = afr_data_subvol_get (local->inode, this,
+							 NULL, NULL);
+	}
+	if (local->parent)
+		parent_read_subvol = afr_data_subvol_get (local->parent, this,
+							  NULL, NULL);
+	if (local->parent2)
+		parent2_read_subvol = afr_data_subvol_get (local->parent2, this,
+							   NULL, NULL);
+
+	local->op_ret = -1;
+	local->op_errno = afr_final_errno (local, priv);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret < 0) {
+			if (local->inode)
+				afr_inode_read_subvol_reset (local->inode,
+							     this);
+			if (local->parent)
+				afr_inode_read_subvol_reset (local->parent,
+							     this);
+			if (local->parent2)
+				afr_inode_read_subvol_reset (local->parent2,
+							     this);
+			continue;
+		}
+
+		if (local->op_ret == -1) {
+			local->op_ret = local->replies[i].op_ret;
+			local->op_errno = local->replies[i].op_errno;
+
+			local->cont.dir_fop.buf =
+				local->replies[i].poststat;
+			local->cont.dir_fop.preparent =
+				local->replies[i].preparent;
+			local->cont.dir_fop.postparent =
+				local->replies[i].postparent;
+			local->cont.dir_fop.prenewparent =
+				local->replies[i].preparent2;
+			local->cont.dir_fop.postnewparent =
+				local->replies[i].postparent2;
+			if (local->replies[i].xdata)
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			continue;
+		}
+
+		if (i == inode_read_subvol) {
+			local->cont.dir_fop.buf =
+				local->replies[i].poststat;
+			if (local->replies[i].xdata) {
+				if (local->xdata_rsp)
+					dict_unref (local->xdata_rsp);
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			}
+		}
+
+		if (i == parent_read_subvol) {
+			local->cont.dir_fop.preparent =
+				local->replies[i].preparent;
+			local->cont.dir_fop.postparent =
+				local->replies[i].postparent;
+		}
+
+		if (i == parent2_read_subvol) {
+			local->cont.dir_fop.prenewparent =
+				local->replies[i].preparent2;
+			local->cont.dir_fop.postnewparent =
+				local->replies[i].postparent2;
+		}
+	}
+}
+
+
+static void
+__afr_dir_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
+		      int op_ret, int op_errno, struct iatt *poststat,
+		      struct iatt *preparent, struct iatt *postparent,
+		      struct iatt *preparent2, struct iatt *postparent2,
+		      dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+	afr_fd_ctx_t *fd_ctx = NULL;
 
         local = frame->local;
+	fd_ctx = local->fd_ctx;
+
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+
+	if (op_ret >= 0) {
+		if (poststat)
+			local->replies[child_index].poststat = *poststat;
+		if (preparent)
+			local->replies[child_index].preparent = *preparent;
+		if (postparent)
+			local->replies[child_index].postparent = *postparent;
+		if (preparent2)
+			local->replies[child_index].preparent2 = *preparent2;
+		if (postparent2)
+			local->replies[child_index].postparent2 = *postparent2;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+
+		if (fd_ctx)
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+	} else {
+		if (op_errno != ENOTEMPTY)
+			afr_transaction_fop_failed (frame, this, child_index);
+		if (fd_ctx)
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+	}
+
+        return;
+}
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.create.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.create.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.create.buf;
-                }
-
-                AFR_STACK_UNWIND (create, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.create.fd,
-                                  local->cont.create.inode,
-                                  unwind_buf, &local->cont.create.preparent,
-                                  &local->cont.create.postparent);
+
+static int
+__afr_dir_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int op_ret, int op_errno, struct iatt *buf,
+		     struct iatt *preparent, struct iatt *postparent,
+		     struct iatt *preparent2, struct iatt *postparent2,
+                     dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
+
+        local = frame->local;
+
+	LOCK (&frame->lock);
+	{
+		__afr_dir_write_fill (frame, this, child_index, op_ret,
+				      op_errno, buf, preparent, postparent,
+				      preparent2, postparent2, xdata);
+	}
+	UNLOCK (&frame->lock);
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+		__afr_dir_write_finalize (frame, this);
+
+		if (afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
+
+		afr_mark_entry_pending_changelog (frame, this);
+
+                local->transaction.resume (frame, this);
         }
 
         return 0;
@@ -114,202 +253,236 @@ afr_create_unwind (call_frame_t *frame, xlator_t *this)
 
 
 int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent)
+afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
+				  xlator_t *this, int op_ret, int op_errno,
+                                  dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx = NULL;
-        int             ret = 0;
-        int             call_count = -1;
-        int             child_index = -1;
-        int32_t         *fresh_children = NULL;
+        int call_count = 0;
 
-        local = frame->local;
-        priv  = this->private;
+        call_count = afr_frame_return (frame);
 
-        child_index = (long) cookie;
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
 
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+        return 0;
+}
 
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
 
-                        ret = afr_fd_ctx_set (this, fd);
+void
+afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *new_frame  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_local_t   *new_local  = NULL;
+        afr_private_t *priv       = NULL;
+        dict_t        *xattr      = NULL;
+        int32_t       **changelog = NULL;
+        int           i           = 0;
+	int           idx         = 0;
+        int           op_errno    = ENOMEM;
+	unsigned char *pending    = NULL;
+	int           call_count   = 0;
 
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not set ctx on fd=%p", fd);
+        local = frame->local;
+        priv = this->private;
 
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
 
-                        ret = fd_ctx_get (fd, this, &ctx);
+	new_local = AFR_FRAME_INIT (new_frame, op_errno);
+	if (!new_local)
+		goto out;
 
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not get fd ctx for fd=%p", fd);
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
+        changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog)
+                goto out;
 
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        xattr = dict_new ();
+	if (!xattr)
+		goto out;
 
-                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-                        fd_ctx->flags                  = local->cont.create.flags;
+	idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
 
-                        if (local->success_count == 0)
-                                local->cont.create.buf        = *buf;
+	pending = alloca0 (priv->child_count);
 
-                        if (child_index == local->read_child_index) {
-                                local->cont.create.read_child_buf = *buf;
-                                local->cont.create.preparent      = *preparent;
-                                local->cont.create.postparent     = *postparent;
-                        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->transaction.pre_op[i] &&
+		    !local->transaction.failed_subvols[i]) {
+			call_count ++;
+			continue;
+		}
 
-                        local->cont.create.inode = inode;
+		changelog[i][idx] = hton32(1);
+		pending[i] = 1;
+	}
 
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
+        new_local->pending = changelog;
+        uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+        new_local->loc.inode = inode_ref (local->inode);
 
-                local->op_errno = op_errno;
-        }
 
-unlock:
-        UNLOCK (&frame->lock);
+	afr_set_pending_dict (priv, xattr, changelog);
 
-        call_count = afr_frame_return (frame);
+        new_local->call_count = call_count;
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
+        for (i = 0; i < priv->child_count; i++) {
+		if (pending[i])
+                        continue;
 
-                local->transaction.resume (frame, this);
+                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->xattrop,
+                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
+                                   xattr, NULL);
+		if (!--call_count)
+			break;
         }
 
-        return 0;
+        new_frame = NULL;
+out:
+        if (new_frame)
+                AFR_STACK_DESTROY (new_frame);
+	if (xattr)
+		dict_unref (xattr);
+        return;
 }
 
 
-int
-afr_create_wind (call_frame_t *frame, xlator_t *this)
+void
+afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+	int pre_op_count = 0;
+	int failed_count = 0;
 
         local = frame->local;
-        priv = this->private;
+        priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+        if (local->op_ret < 0)
+		return;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+	if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD)
+		return;
 
-        local->call_count = call_count;
+	pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+	failed_count = AFR_COUNT (local->transaction.failed_subvols,
+				  priv->child_count);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_create_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->create,
-                                           &local->loc,
-                                           local->cont.create.flags,
-                                           local->cont.create.mode,
-                                           local->cont.create.fd,
-                                           local->cont.create.params);
-                        if (!--call_count)
-                                break;
-                }
-        }
+	if (pre_op_count == priv->child_count && !failed_count)
+		return;
 
-        return 0;
+        afr_mark_new_entry_changelog (frame, this);
+
+        return;
 }
 
 
+/* {{{ create */
+
 int
-afr_create_done (call_frame_t *frame, xlator_t *this)
+afr_create_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = NULL;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
         local = frame->local;
 
-        local->transaction.unwind (frame, this);
+	main_frame = afr_transaction_detach_fop_frame (frame);
 
-        AFR_STACK_DESTROY (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (create, main_frame, local->op_ret, local->op_errno,
+			  local->cont.create.fd, local->inode,
+			  &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     fd_t *fd, inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
-        afr_private_t  *priv  = NULL;
-        afr_local_t    *local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int             ret = -1;
-        int             op_errno = 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+
+int
+afr_create_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_create_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->create,
+			   &local->loc, local->cont.create.flags,
+			   local->cont.create.mode, local->umask,
+			   local->cont.create.fd, local->xdata_req);
+        return 0;
+}
+
+
+int
+afr_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+	    mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(create,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+	local->fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!local->fd_ctx)
+		goto out;
 
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
+
+        local->op                = GF_FOP_CREATE;
         local->cont.create.flags = flags;
         local->cont.create.mode  = mode;
         local->cont.create.fd    = fd_ref (fd);
-        if (params)
-                local->cont.create.params = dict_ref (params);
+        local->umask  = umask;
+
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_create_wind;
-        local->transaction.done   = afr_create_done;
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_create_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_create_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -319,18 +492,30 @@ afr_create (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (create, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+            op_errno = -ret;
+            goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL, NULL);
         return 0;
 }
 
@@ -343,33 +528,17 @@ afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.mknod.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mknod.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mknod.buf;
-                }
-
-                AFR_STACK_UNWIND (mknod, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.mknod.inode,
-                                  unwind_buf, &local->cont.mknod.preparent,
-                                  &local->cont.mknod.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (mknod, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -378,162 +547,74 @@ int
 afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mknod.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mknod.read_child_buf = *buf;
-                                local->cont.mknod.preparent      = *preparent;
-                                local->cont.mknod.postparent     = *postparent;
-                        }
-
-                        local->cont.mknod.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
-int32_t
-afr_mknod_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_mknod_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->mknod,
-                                           &local->loc, local->cont.mknod.mode,
-                                           local->cont.mknod.dev,
-                                           local->cont.mknod.params);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->mknod,
+			   &local->loc, local->cont.mknod.mode,
+			   local->cont.mknod.dev, local->umask,
+			   local->xdata_req);
         return 0;
 }
 
-
 int
-afr_mknod_done (call_frame_t *frame, xlator_t *this)
+afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t dev, mode_t umask, dict_t *xdata)
 {
-        afr_local_t * local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int
-afr_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t dev, dict_t *params)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(mknod,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
+        local->op               = GF_FOP_MKNOD;
         local->cont.mknod.mode  = mode;
         local->cont.mknod.dev   = dev;
-        if (params)
-                local->cont.mknod.params = dict_ref (params);
+        local->umask = umask;
+
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_mknod_wind;
-        local->transaction.done   = afr_mknod_done;
+        local->transaction.wind   = afr_mknod_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_mknod_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -543,18 +624,30 @@ afr_mknod (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (mknod, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
         return 0;
 }
 
@@ -568,33 +661,17 @@ afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.mkdir.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mkdir.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mkdir.buf;
-                }
-
-                AFR_STACK_UNWIND (mkdir, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.mkdir.inode,
-                                  unwind_buf, &local->cont.mkdir.preparent,
-                                  &local->cont.mkdir.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (mkdir, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -603,162 +680,73 @@ int
 afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret           = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mkdir.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mkdir.read_child_buf = *buf;
-                                local->cont.mkdir.preparent      = *preparent;
-                                local->cont.mkdir.postparent     = *postparent;
-                        }
-
-                        local->cont.mkdir.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno         = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_mkdir_wind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->mkdir,
-                                           &local->loc, local->cont.mkdir.mode,
-                                           local->cont.mkdir.params);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->mkdir, &local->loc,
+			   local->cont.mkdir.mode, local->umask,
+			   local->xdata_req);
         return 0;
 }
 
 
 int
-afr_mkdir_done (call_frame_t *frame, xlator_t *this)
+afr_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+	   mode_t umask, dict_t *xdata)
 {
-        afr_local_t * local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dict_t *params)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(mkdir,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
         local->cont.mkdir.mode  = mode;
-        if (params)
-                local->cont.mkdir.params = dict_ref (params);
+        local->umask = umask;
 
-        local->transaction.fop    = afr_mkdir_wind;
-        local->transaction.done   = afr_mkdir_done;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_MKDIR;
+        local->transaction.wind   = afr_mkdir_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_mkdir_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -768,19 +756,30 @@ afr_mkdir (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (mkdir, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
         return 0;
 }
 
@@ -794,33 +793,17 @@ afr_link_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.link.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.link.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.link.buf;
-                }
-
-                AFR_STACK_UNWIND (link, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.link.inode,
-                                  unwind_buf, &local->cont.link.preparent,
-                                  &local->cont.link.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (link, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -829,158 +812,72 @@ int
 afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0) {
-                                local->cont.link.buf        = *buf;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.link.read_child_buf = *buf;
-                                local->cont.link.preparent      = *preparent;
-                                local->cont.link.postparent     = *postparent;
-                        }
-
-                        local->cont.link.inode    = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_link_wind (call_frame_t *frame, xlator_t *this)
+afr_link_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->link,
-                                           &local->loc,
-                                           &local->newloc);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->link,
+			   &local->loc, &local->newloc, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_link_done (call_frame_t *frame, xlator_t *this)
+afr_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+	  dict_t *xdata)
 {
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int
-afr_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(link,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc,    oldloc);
         loc_copy (&local->newloc, newloc);
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+	local->inode = inode_ref (oldloc->inode);
+	local->parent = inode_ref (newloc->parent);
 
-        local->transaction.fop    = afr_link_wind;
-        local->transaction.done   = afr_link_done;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_LINK;
+
+        local->transaction.wind   = afr_link_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_link_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
@@ -990,18 +887,30 @@ afr_link (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (link, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
         return 0;
 }
 
@@ -1015,33 +924,17 @@ afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.symlink.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.symlink.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.symlink.buf;
-                }
-
-                AFR_STACK_UNWIND (symlink, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.symlink.inode,
-                                  unwind_buf, &local->cont.symlink.preparent,
-                                  &local->cont.symlink.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (symlink, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -1050,163 +943,73 @@ int
 afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, inode_t *inode,
                       struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent)
+                      struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.symlink.buf        = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.symlink.read_child_buf = *buf;
-                                local->cont.symlink.preparent      = *preparent;
-                                local->cont.symlink.postparent     = *postparent;
-                        }
-
-                        local->cont.symlink.inode    = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_symlink_wind (call_frame_t *frame, xlator_t *this)
+afr_symlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->symlink,
-                                           local->cont.symlink.linkpath,
-                                           &local->loc,
-                                           local->cont.symlink.params);
-
-                        if (!--call_count)
-                                break;
-
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_symlink_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->symlink,
+			   local->cont.symlink.linkpath, &local->loc,
+			   local->umask, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkpath, loc_t *loc, dict_t *params)
+afr_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+	     loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(symlink,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
         local->cont.symlink.linkpath = gf_strdup (linkpath);
-        if (params)
-                local->cont.symlink.params = dict_ref (params);
+        local->umask = umask;
+
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_symlink_wind;
-        local->transaction.done   = afr_symlink_done;
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_SYMLINK;
+        local->transaction.wind   = afr_symlink_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_symlink_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -1216,18 +1019,30 @@ afr_symlink (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
-        if (ret  < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (symlink, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
+			  NULL, NULL);
         return 0;
 }
 
@@ -1240,35 +1055,19 @@ afr_rename_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.rename.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.rename.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.rename.buf;
-                }
-
-                AFR_STACK_UNWIND (rename, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  unwind_buf,
-                                  &local->cont.rename.preoldparent,
-                                  &local->cont.rename.postoldparent,
-                                  &local->cont.rename.prenewparent,
-                                  &local->cont.rename.postnewparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (rename, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent,
+			  &local->cont.dir_fop.prenewparent,
+			  &local->cont.dir_fop.postnewparent, local->xdata_rsp);
         return 0;
 }
 
@@ -1277,146 +1076,75 @@ int
 afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                      struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent)
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        int call_count = -1;
-        int child_index = -1;
-
-        local = frame->local;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != ENOTEMPTY)
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-
-                                if (buf) {
-                                        local->cont.rename.buf = *buf;
-                                }
-
-                                local->success_count++;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.rename.read_child_buf = *buf;
-
-                                local->cont.rename.preoldparent  = *preoldparent;
-                                local->cont.rename.postoldparent = *postoldparent;
-                                local->cont.rename.prenewparent  = *prenewparent;
-                                local->cont.rename.postnewparent = *postnewparent;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preoldparent, postoldparent, prenewparent,
+				    postnewparent, xdata);
 }
 
 
-int32_t
-afr_rename_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_rename_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_rename_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->rename,
-                                           &local->loc,
-                                           &local->newloc);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_rename_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
 
-        AFR_STACK_DESTROY (frame);
+	local = frame->local;
+	priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->rename,
+			   &local->loc, &local->newloc, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc)
+afr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+	    dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
+        int                     nlockee                 = 0;
 
         priv = this->private;
 
         QUORUM_CHECK(rename,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
+        if (!transaction_frame)
                 op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc,    oldloc);
         loc_copy (&local->newloc, newloc);
 
-        local->read_child_index = afr_inode_get_read_ctx (this, oldloc->inode, NULL);
+	local->inode = inode_ref (oldloc->inode);
+	local->parent = inode_ref (oldloc->parent);
+	local->parent2 = inode_ref (newloc->parent);
+
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_rename_wind;
-        local->transaction.done   = afr_rename_done;
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_RENAME;
+        local->transaction.wind   = afr_rename_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_rename_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
@@ -1431,19 +1159,52 @@ afr_rename (call_frame_t *frame, xlator_t *this,
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (oldloc->path);
         local->transaction.new_basename = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.new_parent_loc,
+                                     local->transaction.new_basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
+                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                             &local->newloc,
+                                             NULL,
+                                             priv->child_count);
+                if (ret)
+                        goto out;
 
-                AFR_STACK_UNWIND (rename, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+                nlockee++;
         }
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
 
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
+        }
+
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL, NULL);
         return 0;
 }
 
@@ -1459,22 +1220,13 @@ afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (unlink, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.unlink.preparent,
-                                  &local->cont.unlink.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (unlink, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -1482,140 +1234,71 @@ afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == local->read_child_index) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret   = op_ret;
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
-int32_t
-afr_unlink_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_unlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->unlink,
-                                           &local->loc);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int32_t
-afr_unlink_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->unlink,
+			   &local->loc, local->xflag, local->xdata_req);
         return 0;
 }
 
 
-int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-            loc_t *loc)
+int
+afr_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+	    dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(unlink,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
+        local->xflag = xflag;
 
-        local->transaction.fop    = afr_unlink_wind;
-        local->transaction.done   = afr_unlink_done;
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
+
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_UNLINK;
+        local->transaction.wind   = afr_unlink_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_unlink_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -1625,18 +1308,29 @@ afr_unlink (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (unlink, frame, -1, op_errno,
-                                  NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1654,22 +1348,13 @@ afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (rmdir, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.rmdir.preparent,
-                                  &local->cont.rmdir.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (rmdir, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -1677,142 +1362,73 @@ afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno) && (op_errno != ENOTEMPTY))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_rmdir_wind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->rmdir,
-                                           &local->loc, local->cont.rmdir.flags);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_rmdir_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->rmdir,
+			   &local->loc, local->cont.rmdir.flags, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int flags)
+afr_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	   dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
+        int                     nlockee                 = 0;
 
         priv = this->private;
 
         QUORUM_CHECK(rmdir,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
 
-        local->cont.rmdir.flags = flags;
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
+
+        local->cont.rmdir.flags = flags;
 
-        local->transaction.fop    = afr_rmdir_wind;
-        local->transaction.done   = afr_rmdir_done;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_RMDIR;
+        local->transaction.wind   = afr_rmdir_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_rmdir_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -1822,17 +1438,41 @@ afr_rmdir (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->loc,
+                                     NULL,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        nlockee++;
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
+
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index 0290c6350..02f0a3682 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_WRITE_H__
@@ -23,38 +14,34 @@
 int32_t
 afr_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params);
+            mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, dict_t *params);
+	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *params);
+	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc);
+	    loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t
 afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags);
+	   loc_t *loc, int flags, dict_t *xdata);
 
 int32_t
 afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc);
+	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
 afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc);
+	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int
 afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, dict_t *params);
-
-int32_t
-afr_setdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t flags, dir_entry_t *entries, int32_t count);
+	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index c46192702..01e078c13 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -44,235 +35,153 @@
 #include "compat-errno.h"
 #include "compat.h"
 
-/**
- * Common algorithm for inode read calls:
- *
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: access, stat, fstat, readlink, getxattr
- */
+#include "afr-transaction.h"
+
 
 /* {{{ access */
 
-int32_t
-afr_access_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno)
+int
+afr_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+        afr_local_t     *local = NULL;
 
         local = frame->local;
 
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.access.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                unwind = 0;
-
-                STACK_WIND_COOKIE (frame, afr_access_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->access,
-                                   &local->loc, local->cont.access.mask);
-        }
-
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (access, frame, op_ret, op_errno);
-        }
+	AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+int
+afr_access_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t   *priv      = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        afr_local_t     *local     = NULL;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+
+	priv = this->private;
+	local = frame->local;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (access, frame, local->op_ret,
+				  local->op_errno, 0);
+		return 0;
+	}
+
+        STACK_WIND_COOKIE (frame, afr_access_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->access,
+                           &local->loc, local->cont.access.mask,
+			   local->xdata_req);
+	return 0;
+}
 
+int
+afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	    int mask, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+	int op_errno = 0;
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.access.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        loc_copy (&local->loc, loc);
-        local->cont.access.mask = mask;
+	local->op = GF_FOP_ACCESS;
+	loc_copy (&local->loc, loc);
+	local->cont.access.mask = mask;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        STACK_WIND_COOKIE (frame, afr_access_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->access,
-                           loc, mask);
+	afr_read_txn (frame, this, loc->inode, afr_access_wind,
+		      AFR_METADATA_TRANSACTION);
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (access, frame, -1, op_errno);
+	AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
-
 /* }}} */
 
 /* {{{ stat */
 
-int32_t
+int
 afr_stat_cbk (call_frame_t *frame, void *cookie,
               xlator_t *this, int32_t op_ret, int32_t op_errno,
-              struct iatt *buf)
+              struct iatt *buf, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
-
-        read_child = (long) cookie;
+        afr_local_t     *local = NULL;
 
         local = frame->local;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.stat.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-                unwind = 0;
-
-                STACK_WIND_COOKIE (frame, afr_stat_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->stat,
-                                   &local->loc);
-        }
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
-        }
+	AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+int
+afr_stat_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t   *priv      = NULL;
-        afr_local_t     *local     = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+
+	priv = this->private;
+	local = frame->local;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
+				  0, 0);
+		return 0;
+	}
+
+        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->stat,
+                           &local->loc, local->xdata_req);
+	return 0;
+}
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+int
+afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+	int op_errno = 0;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.stat.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-        loc_copy (&local->loc, loc);
+	local->op = GF_FOP_STAT;
+	loc_copy (&local->loc, loc);
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->stat,
-                           loc);
+	afr_read_txn (frame, this, loc->inode, afr_stat_wind,
+		      AFR_DATA_TRANSACTION);
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -282,121 +191,76 @@ out:
 
 /* {{{ fstat */
 
-int32_t
+int
 afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf)
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        xlator_t        **children      = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+        afr_local_t     *local = NULL;
 
         local = frame->local;
 
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.fstat.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                unwind = 0;
+	AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
 
-                STACK_WIND_COOKIE (frame, afr_fstat_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->fstat,
-                                   local->fd);
-        }
+        return 0;
+}
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
-        }
 
-        return 0;
+int
+afr_fstat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+
+	priv = this->private;
+	local = frame->local;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (fstat, frame, local->op_ret, local->op_errno,
+				  0, 0);
+		return 0;
+	}
+
+        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->fstat,
+                           local->fd, local->xdata_req);
+	return 0;
 }
 
 
 int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
-           fd_t *fd)
+           fd_t *fd, dict_t *xdata)
 {
-        afr_private_t   *priv      = NULL;
-        afr_local_t     *local     = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = 0;
-        int             ret        = -1;
+        afr_local_t *local = NULL;
+	int op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (fd, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        children = priv->children;
-
-        VALIDATE_OR_GOTO (fd->inode, out);
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+	local->op = GF_FOP_FSTAT;
+	local->fd = fd_ref (fd);
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode,
-                                             local->fresh_children);
+        afr_fix_open (fd, this);
 
+	afr_read_txn (frame, this, fd->inode, afr_fstat_wind,
+		      AFR_DATA_TRANSACTION);
 
-
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.fstat.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        local->fd = fd_ref (fd);
-
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->fstat,
-                           fd);
-
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -405,114 +269,77 @@ out:
 
 /* {{{ readlink */
 
-int32_t
+int
 afr_readlink_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  const char *buf, struct iatt *sbuf)
+                  const char *buf, struct iatt *sbuf, dict_t *xdata)
 {
-        afr_private_t * priv                  = NULL;
-        afr_local_t *   local                 = NULL;
-        xlator_t **     children              = NULL;
-        int             unwind                = 1;
-        int32_t         *last_index           = NULL;
-        int32_t         next_call_child       = -1;
-        int32_t         read_child            = -1;
-        int32_t         *fresh_children        = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+	afr_local_t *local = NULL;
 
-        local = frame->local;
+	local = frame->local;
 
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.readlink.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->readlink,
-                                   &local->loc,
-                                   local->cont.readlink.size);
-        }
-
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf);
-        }
+	AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno,
+			  buf, sbuf, xdata);
+	return 0;
+}
 
-        return 0;
+int
+afr_readlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readlink, frame, local->op_ret,
+				  local->op_errno, 0, 0, 0);
+		return 0;
+	}
+
+	STACK_WIND_COOKIE (frame, afr_readlink_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->readlink,
+			   &local->loc, local->cont.readlink.size,
+			   local->xdata_req);
+	return 0;
 }
 
 
-int32_t
+int
 afr_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size)
+              loc_t *loc, size_t size, dict_t *xdata)
 {
-        afr_private_t   *priv      = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        afr_local_t     *local     = NULL;
+        afr_local_t   * local      = NULL;
         int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readlink.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
+	local->op = GF_FOP_READLINK;
         loc_copy (&local->loc, loc);
+        local->cont.readlink.size = size;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        local->cont.readlink.size       = size;
-
-        STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->readlink,
-                           loc, size);
+	afr_read_txn (frame, this, loc->inode, afr_readlink_wind,
+		      AFR_DATA_TRANSACTION);
 
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
         return 0;
+out:
+	AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
+
+	return 0;
 }
 
 
@@ -526,7 +353,7 @@ struct _xattr_key {
 };
 
 
-void
+int
 __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
                      void *data)
 {
@@ -538,18 +365,19 @@ __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
 
                 xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
                 if (!xkey)
-                        return;
+                        return -1;
 
                 xkey->key = key;
                 INIT_LIST_HEAD (&xkey->list);
 
                 list_add_tail (&xkey->list, list);
         }
+        return 0;
 }
 
 
 void
-__filter_xattrs (dict_t *dict)
+afr_filter_xattrs (dict_t *dict)
 {
         struct list_head   keys = {0,};
         struct _xattr_key *key  = NULL;
@@ -570,71 +398,157 @@ __filter_xattrs (dict_t *dict)
 }
 
 
-
-int32_t
+int
 afr_getxattr_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *dict)
+                  dict_t *dict, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+        afr_local_t *local = NULL;
 
         local = frame->local;
 
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->getxattr,
-                                   &local->loc,
-                                   local->cont.getxattr.name);
-        }
+	if (dict)
+		afr_filter_xattrs (dict);
 
-out:
-        if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
-
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
-        }
+	AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
 
+
+int
+afr_getxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+				  local->op_errno, NULL, NULL);
+		return 0;
+	}
+
+	STACK_WIND_COOKIE (frame, afr_getxattr_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->getxattr,
+			   &local->loc, local->cont.getxattr.name,
+			   local->xdata_req);
+	return 0;
+}
+
+
 int32_t
-afr_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict)
+afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
+                     dict_t *dict, dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 int32_t
+afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *dict, dict_t *xdata)
+{
+        afr_local_t     *local                  = NULL;
+        afr_private_t   *priv                   = NULL;
+        xlator_t        **children              = NULL;
+        dict_t          *xattr                  = NULL;
+        char            *tmp_report             = NULL;
+        char            lk_summary[1024]        = {0,};
+        int             serz_len                = 0;
+        int32_t         callcnt                 = 0;
+        long int        cky                     = 0;
+        int             ret                     = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->replies[cky].op_errno = op_errno;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+                if (local->dict) {
+                        ret = dict_get_str (dict, local->cont.getxattr.name,
+                                            &tmp_report);
+                        if (ret)
+                                goto unlock;
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
+                        if (ret)
+                                goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                xattr = dict_new ();
+                if (!xattr) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       lk_summary,
+                                                       &serz_len, '\n');
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error serializing dictionary");
+                        goto unwind;
+                }
+                if (serz_len == -1)
+                        snprintf (lk_summary, sizeof (lk_summary),
+                                  "No locks cleared.");
+                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
+                                       gf_strdup (lk_summary));
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting dictionary");
+                        goto unwind;
+                }
+
+        unwind:
+                // Updating child_errno with more recent 'events'
+                op_errno = afr_final_errno (local, priv);
+
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
+                                  xdata);
+                if (xattr)
+                        dict_unref (xattr);
+        }
+
+        return ret;
+}
+
+int32_t
 afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                        dict_t *dict)
+                        dict_t *dict, dict_t *xdata)
 {
         afr_local_t     *local                  = NULL;
         afr_private_t   *priv                   = NULL;
@@ -656,6 +570,8 @@ afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
         LOCK (&frame->lock);
         {
                 callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->replies[cky].op_errno = op_errno;
 
                 if (!local->dict)
                         local->dict = dict_new ();
@@ -664,14 +580,12 @@ afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
                                             &tmp_report);
                         if (ret)
                                 goto unlock;
-                        ret = dict_set_str (local->dict,
-                                            children[cky]->name,
-                                            tmp_report);
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
                         if (ret)
                                 goto unlock;
                 }
-                if (op_ret == -1)
-                        local->child_errno[cky] = op_errno;
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -708,16 +622,9 @@ unlock:
 
         unwind:
                 // Updating child_errno with more recent 'events'
-                local->child_errno[cky] = op_errno;
-                op_errno = afr_resultant_errno_get (NULL, local->child_errno,
-                                                    priv->child_count);
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
-
-                if (local->dict)
-                        dict_unref (local->dict);
+                op_errno = afr_final_errno (local, priv);
 
-                if (local->child_errno)
-                        GF_FREE (local->child_errno);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
                 if (xattr)
                         dict_unref (xattr);
@@ -732,7 +639,7 @@ unlock:
 int32_t
 afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict)
+                            dict_t *dict, dict_t *xdata)
 {
         afr_private_t  *priv            = NULL;
         afr_local_t    *local           = NULL;
@@ -765,20 +672,381 @@ afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
                                    children[curr_call_child],
                                    children[curr_call_child]->fops->getxattr,
                                    &local->loc,
-                                   local->cont.getxattr.name);
+                                   local->cont.getxattr.name,
+                                   NULL);
         }
 
  unwind:
         if (unwind)
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
+                                  NULL);
+
+        return 0;
+}
+
+int32_t
+afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                           xlator_t *this, int32_t op_ret, int32_t op_errno,
+                           dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len == 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (getxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
 
         return 0;
 }
 
 int32_t
+afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len <= 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
+
+int32_t
+afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret < 0) {
+                        local->op_errno = op_errno;
+                } else {
+                        local->op_ret = op_ret;
+                        if (!local->xdata_rsp && xdata)
+                                local->xdata_rsp = dict_ref (xdata);
+                }
+
+                if (!dict || (op_ret < 0))
+                        goto out;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+
+                if (local->dict) {
+                        ret = dict_get_str (dict,
+                                            local->cont.getxattr.name,
+                                            &xattr);
+                        if (ret)
+                                goto out;
+
+                        xattr = gf_strdup (xattr);
+
+                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                        local->cont.getxattr.name, cky);
+                        ret = dict_set_dynstr (local->dict,
+                                               xattr_cky, xattr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Cannot set xattr cookie key");
+                                goto out;
+                        }
+
+                        local->cont.getxattr.xattr_len
+                                += strlen (xattr) + 1;
+                }
+        }
+out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->cont.getxattr.xattr_len)
+                        goto unwind;
+
+                nxattr = dict_new ();
+                if (!nxattr)
+                        goto unwind;
+
+                /* extra bytes for decorations (brackets and <>'s) */
+                padding += strlen (this->name)
+                        + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
+
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
+
+                if (!xattr_serz)
+                        goto unwind;
+
+                /* the xlator info */
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
+
+                /* actual series of pathinfo */
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz
+                                                       + strlen (xattr_serz),
+                                                       &tlen, ' ');
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
+                        goto unwind;
+                }
+
+                /* closing part */
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
+
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
+                                  local->op_errno, nxattr, local->xdata_rsp);
+
+                if (nxattr)
+                        dict_unref (nxattr);
+        }
+
+        return ret;
+}
+
+int32_t
 afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict)
+                           dict_t *dict, dict_t *xdata)
 {
         afr_local_t *local          = NULL;
         int32_t      callcnt        = 0;
@@ -803,6 +1071,14 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                 {
                         callcnt = --local->call_count;
 
+                        if (op_ret < 0) {
+                                local->op_errno = op_errno;
+                        } else {
+                                local->op_ret = op_ret;
+                                if (!local->xdata_rsp && xdata)
+                                        local->xdata_rsp = dict_ref (xdata);
+                        }
+
                         if (!dict || (op_ret < 0))
                                 goto out;
 
@@ -877,10 +1153,8 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                                 " key in dict");
 
         unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, nxattr);
-
-                if (local->dict)
-                        dict_unref (local->dict);
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, nxattr, local->xdata_rsp);
 
                 if (nxattr)
                         dict_unref (nxattr);
@@ -889,120 +1163,264 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
         return ret;
 }
 
+static int
+afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
+{
+        int ret = 0;
+
+        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+                ret = gf_get_max_stime (THIS, data, key, value);
+
+        return ret;
+}
+
+int32_t
+afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (!dict || (op_ret < 0)) {
+                        local->op_errno = op_errno;
+                        goto cleanup;
+                }
+
+                if (!local->dict)
+                        local->dict = dict_copy_with_ref (dict, NULL);
+                else
+                        dict_foreach (dict, afr_aggregate_stime_xattr,
+                                      local->dict);
+                local->op_ret = 0;
+        }
+
+cleanup:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, local->dict, xdata);
+        }
+
+out:
+        return 0;
+}
+
+
 static gf_boolean_t
-afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk)
+afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
+                      gf_boolean_t is_fgetxattr)
 {
         gf_boolean_t    is_spl = _gf_true;
 
         GF_ASSERT (cbk);
-        if (!cbk) {
+        if (!cbk || !name) {
                 is_spl = _gf_false;
                 goto out;
         }
 
-        if (!strcmp (name, GF_XATTR_PATHINFO_KEY))
-                *cbk = afr_getxattr_pathinfo_cbk;
-
-        else if (!strcmp (name, GF_XATTR_CLRLK_CMD))
-                *cbk = afr_getxattr_clrlk_cbk;
-        else
+        if (!strcmp (name, GF_XATTR_PATHINFO_KEY) ||
+	    !strcmp (name, GF_XATTR_USER_PATHINFO_KEY)) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_pathinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_pathinfo_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
+                             strlen (GF_XATTR_CLRLK_CMD))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_clrlk_cbk;
+                } else {
+                        *cbk = afr_getxattr_clrlk_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                             strlen (GF_XATTR_LOCKINFO_KEY))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_lockinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_lockinfo_cbk;
+                }
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+                *cbk = afr_common_getxattr_stime_cbk;
+        } else {
                 is_spl = _gf_false;
+        }
 
 out:
         return is_spl;
 }
 
 static void
-afr_getxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
-                               const char *name, loc_t *loc,
-                               fop_getxattr_cbk_t cbk)
+afr_getxattr_all_subvols (xlator_t *this, call_frame_t *frame,
+			  const char *name, loc_t *loc,
+			  fop_getxattr_cbk_t cbk)
 {
         afr_private_t   *priv           = NULL;
         afr_local_t     *local          = NULL;
-        xlator_t        **children      = NULL;
         int             i               = 0;
+        int             call_count      = 0;
 
         priv     = this->private;
-        children = priv->children;
 
         local = frame->local;
-        local->call_count = priv->child_count;
+        //local->call_count set in afr_local_init
+        call_count = local->call_count;
+
+        //If up-children count is 0, afr_local_init would have failed already
+        //and the call would have unwound so not handling it here.
 
         for (i = 0; i < priv->child_count; i++) {
-                STACK_WIND_COOKIE (frame, cbk,
-                                   (void *) (long) i,
-                                   children[i], children[i]->fops->getxattr,
-                                   loc, name);
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, cbk,
+                                           (void *) (long) i, priv->children[i],
+                                           priv->children[i]->fops->getxattr,
+                                           loc, name, NULL);
+                        if (!--call_count)
+                                break;
+                }
         }
         return;
 }
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *name)
+              loc_t *loc, const char *name, dict_t *xdata)
 {
         afr_private_t           *priv         = NULL;
         xlator_t                **children    = NULL;
-        int                     call_child    = 0;
         afr_local_t             *local        = NULL;
         xlator_list_t           *trav         = NULL;
         xlator_t                **sub_volumes = NULL;
         int                     i             = 0;
         int32_t                 op_errno      = 0;
-        int32_t                 read_child    = -1;
         int                     ret           = -1;
         fop_getxattr_cbk_t      cbk           = NULL;
+        int                     afr_xtime_gauge[MCNT_MAX] = {0,};
 
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
 
         children = priv->children;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+        loc_copy (&local->loc, loc);
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+	local->op = GF_FOP_GETXATTR;
+
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+        if (!name)
+                goto no_name;
+
+        local->cont.getxattr.name = gf_strdup (name);
+
+	if (!local->cont.getxattr.name) {
+		op_errno = ENOMEM;
+		goto out;
+	}
+
+        if (!strncmp (name, AFR_XATTR_PREFIX,
+                      strlen (AFR_XATTR_PREFIX))) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: no data present for key %s",
+                        loc->path, name);
+                op_errno = ENODATA;
                 goto out;
+        }
+        if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
 
-        loc_copy (&local->loc, loc);
-        if (name)
-                local->cont.getxattr.name = gf_strdup (name);
+                local->marker.call_count = priv->child_count;
 
+                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                for (i = 0, trav = this->children; trav ;
+                     trav = trav->next, i++) {
+
+                        *(sub_volumes + i)  = trav->xlator;
+                }
+
+                if (cluster_getmarkerattr (frame, this, loc, name,
+                                           local, afr_getxattr_unwind,
+                                           sub_volumes,
+                                           priv->child_count,
+                                           MARKER_UUID_TYPE,
+                                           marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
 
-        if (name) {
-                if (!strncmp (name, AFR_XATTR_PREFIX,
-                              strlen (AFR_XATTR_PREFIX))) {
                         gf_log (this->name, GF_LOG_INFO,
-                                "%s: no data present for key %s",
+                                "%s: failed to get marker attr (%s)",
                                 loc->path, name);
-                        op_errno = ENODATA;
+                        op_errno = EINVAL;
                         goto out;
                 }
-                if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-                    && (-1 == frame->root->pid)) {
 
+                return 0;
+        }
+
+        /*
+         * if we are doing getxattr with pathinfo as the key then we
+         * collect information from all childs
+         */
+        if (afr_is_special_xattr (name, &cbk, 0)) {
+                afr_getxattr_all_subvols (this, frame, name, loc, cbk);
+                return 0;
+        }
+
+        if (XATTR_IS_NODE_UUID (name)) {
+                i = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
+                                   (void *) (long) i,
+                                   children[i],
+                                   children[i]->fops->getxattr,
+                                   loc, name, xdata);
+                return 0;
+        }
+
+        if (*priv->vol_uuid) {
+                if ((match_uuid_local (name, priv->vol_uuid) == 0)
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
                         local->marker.call_count = priv->child_count;
 
-                        sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                        sub_volumes = alloca ( priv->child_count
+                                               * sizeof (xlator_t *));
                         for (i = 0, trav = this->children; trav ;
                              trav = trav->next, i++) {
 
                                 *(sub_volumes + i)  = trav->xlator;
+
                         }
 
-                        if (cluster_getmarkerattr (frame, this, loc, name,
-                                                   local, afr_getxattr_unwind,
+                        /* don't err out on getting ENOTCONN (brick down)
+                         * from a subset of the bricks
+                         */
+                        memcpy (afr_xtime_gauge, marker_xtime_default_gauge,
+                                sizeof (afr_xtime_gauge));
+                        afr_xtime_gauge[MCNT_NOTFOUND] = 0;
+                        afr_xtime_gauge[MCNT_ENOTCONN] = 0;
+                        if (cluster_getmarkerattr (frame, this, loc,
+                                                   name, local,
+                                                   afr_getxattr_unwind,
                                                    sub_volumes,
                                                    priv->child_count,
-                                                   MARKER_UUID_TYPE,
+                                                   MARKER_XTIME_TYPE,
+                                                   afr_xtime_gauge,
                                                    priv->vol_uuid)) {
-
                                 gf_log (this->name, GF_LOG_INFO,
                                         "%s: failed to get marker attr (%s)",
                                         loc->path, name);
@@ -1012,86 +1430,17 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
                         return 0;
                 }
-
-                /*
-                 * if we are doing getxattr with pathinfo as the key then we
-                 * collect information from all childs
-                 */
-                if (afr_is_special_xattr (name, &cbk)) {
-                        afr_getxattr_frm_all_children (this, frame, name,
-                                                       loc, cbk);
-                        return 0;
-                }
-
-                if (XATTR_IS_NODE_UUID (name)) {
-                        i = 0;
-                        STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                           (void *) (long) i,
-                                           children[i],
-                                           children[i]->fops->getxattr,
-                                           loc, name);
-                        return 0;
-                }
-
-                if (*priv->vol_uuid) {
-                        if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                            && (-1 == frame->root->pid)) {
-
-                                local->marker.call_count = priv->child_count;
-
-                                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
-                                for (i = 0, trav = this->children; trav ;
-                                     trav = trav->next, i++) {
-
-                                        *(sub_volumes + i)  = trav->xlator;
-
-                                }
-
-                                if (cluster_getmarkerattr (frame, this, loc,
-                                                           name, local,
-                                                           afr_getxattr_unwind,
-                                                           sub_volumes,
-                                                           priv->child_count,
-                                                           MARKER_XTIME_TYPE,
-                                                           priv->vol_uuid)) {
-                                        gf_log (this->name, GF_LOG_INFO,
-                                                "%s: failed to get marker attr (%s)",
-                                                loc->path, name);
-                                        op_errno = EINVAL;
-                                        goto out;
-                                }
-
-                                return 0;
-                        }
-                }
         }
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+no_name:
 
-        STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->getxattr,
-                           loc, name);
+	afr_read_txn (frame, this, local->loc.inode, afr_getxattr_wind,
+		      AFR_METADATA_TRANSACTION);
 
         ret = 0;
 out:
         if (ret < 0)
-                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -1101,128 +1450,130 @@ out:
 int32_t
 afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
                    xlator_t *this, int32_t op_ret, int32_t op_errno,
-                   dict_t *dict)
+                   dict_t *dict, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
+	afr_local_t *local = NULL;
 
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
+	local = frame->local;
 
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->fgetxattr,
-                                   local->fd,
-                                   local->cont.getxattr.name);
-        }
+	if (dict)
+		afr_filter_xattrs (dict);
 
-out:
-        if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
+	AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict);
-        }
-
-        return 0;
+	return 0;
 }
 
-int32_t
-afr_fgetxattr_unwind (call_frame_t *frame,
-                      int op_ret, int op_errno, dict_t *dict)
-
+int
+afr_fgetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-        AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict);
-        return 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
+				  local->op_errno, NULL, NULL);
+		return 0;
+	}
+
+	STACK_WIND_COOKIE (frame, (void *) (long) subvol, afr_fgetxattr_cbk,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fgetxattr,
+			   local->fd, local->cont.getxattr.name,
+			   local->xdata_req);
+	return 0;
 }
 
-int32_t
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name)
-{
-        afr_private_t   *priv         = NULL;
-        xlator_t        **children    = NULL;
-        int             call_child    = 0;
-        afr_local_t     *local        = NULL;
-        int32_t         op_ret        = -1;
-        int32_t         op_errno      = 0;
-        int32_t         read_child    = -1;
 
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+static void
+afr_fgetxattr_all_subvols (xlator_t *this, call_frame_t *frame,
+			   fop_fgetxattr_cbk_t cbk)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        int             i               = 0;
+        int             call_count      = 0;
 
         priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
 
-        children = priv->children;
+        local = frame->local;
+        //local->call_count set in afr_local_init
+        call_count = local->call_count;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-        frame->local = local;
+        //If up-children count is 0, afr_local_init would have failed already
+        //and the call would have unwound so not handling it here.
 
-        op_ret = afr_local_init (local, priv, &op_errno);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                goto out;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fgetxattr,
+                                           local->fd, local->cont.getxattr.name,
+					   NULL);
+                        if (!--call_count)
+                                break;
+                }
         }
 
-        local->fd = fd_ref (fd);
-        if (name)
-                local->cont.getxattr.name = gf_strdup (name);
+        return;
+}
 
-        /* pathinfo gets handled only in getxattr() */
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+int
+afr_fgetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, const char *name, dict_t *xdata)
+{
+        afr_local_t          *local      = NULL;
+        int32_t               op_errno   = 0;
+        fop_fgetxattr_cbk_t   cbk        = NULL;
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
-        op_ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                op_ret = -1;
-                goto out;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+	local->op = GF_FOP_FGETXATTR;
+        local->fd = fd_ref (fd);
+        if (name) {
+                local->cont.getxattr.name = gf_strdup (name);
+		if (!local->cont.getxattr.name) {
+			op_errno = ENOMEM;
+			goto out;
+		}
+	}
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+        /* pathinfo gets handled only in getxattr(), but we need to handle
+         * lockinfo.
+         * If we are doing fgetxattr with lockinfo as the key then we
+         * collect information from all children.
+         */
+        if (afr_is_special_xattr (name, &cbk, 1)) {
+                afr_fgetxattr_all_subvols (this, frame, cbk);
+                return 0;
         }
 
-        STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->fgetxattr,
-                           fd, name);
+        afr_fix_open (fd, this);
 
-        op_ret = 0;
+	afr_read_txn (frame, this, fd->inode, afr_fgetxattr_wind,
+		      AFR_METADATA_TRANSACTION);
+
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, NULL);
-        }
+	AFR_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -1231,144 +1582,84 @@ out:
 
 /* {{{ readv */
 
-/**
- * read algorithm:
- *
- * if the user has specified a read subvolume, use it
- * otherwise -
- *   use the inode number to hash it to one of the subvolumes, and
- *   read from there (to balance read load)
- *
- * if any of the above read's fail, try the children in sequence
- * beginning at the beginning
- */
-
-int32_t
+int
 afr_readv_cbk (call_frame_t *frame, void *cookie,
                xlator_t *this, int32_t op_ret, int32_t op_errno,
                struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref)
+               struct iobref *iobref, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         *fresh_children  = NULL;
-        int32_t         read_child      = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
+	afr_local_t *local = NULL;
 
-        local = frame->local;
+	local = frame->local;
 
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.readv.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                STACK_WIND_COOKIE (frame, afr_readv_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->readv,
-                                   local->fd, local->cont.readv.size,
-                                   local->cont.readv.offset,
-                                   local->cont.readv.flags);
-        }
+	AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
+			  vector, count, buf, iobref, xdata);
+	return 0;
+}
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-                                  vector, count, buf, iobref);
-        }
 
-        return 0;
+int
+afr_readv_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readv, frame, local->op_ret, local->op_errno,
+				  0, 0, 0, 0, 0);
+		return 0;
+	}
+
+	STACK_WIND_COOKIE (frame, afr_readv_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->readv,
+			   local->fd, local->cont.readv.size,
+			   local->cont.readv.offset, local->cont.readv.flags,
+			   local->xdata_req);
+	return 0;
 }
 
 
-int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags)
+int
+afr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	   off_t offset, uint32_t flags, dict_t *xdata)
 {
-        afr_private_t * priv       = NULL;
         afr_local_t   * local      = NULL;
-        xlator_t **     children   = NULL;
-        int             call_child = 0;
         int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (fd, out);
-
-        priv     = this->private;
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readv.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        local->fd                    = fd_ref (fd);
+	local->op = GF_FOP_READ;
+        local->fd = fd_ref (fd);
+        local->cont.readv.size = size;
+        local->cont.readv.offset = offset;
+        local->cont.readv.flags = flags;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        local->cont.readv.size       = size;
-        local->cont.readv.offset     = offset;
-        local->cont.readv.flags      = flags;
+        afr_fix_open (fd, this);
 
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        STACK_WIND_COOKIE (frame, afr_readv_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->readv,
-                           fd, size, offset, flags);
+	afr_read_txn (frame, this, fd->inode, afr_readv_wind,
+		      AFR_DATA_TRANSACTION);
 
-        ret = 0;
-out:
-        if (ret < 0) {
-                AFR_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL,
-                                  NULL);
-        }
         return 0;
+out:
+	AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
+
+	return 0;
 }
 
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index 5ec7411b1..e4091a793 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_READ_H__
@@ -22,30 +13,30 @@
 
 int32_t
 afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask);
+	    loc_t *loc, int32_t mask, dict_t *xdata);
 
 int32_t
 afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc);
+	  loc_t *loc, dict_t *xdata);
 
 int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd);
+	   fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size);
+	      loc_t *loc, size_t size, dict_t *xdata);
 
 int32_t
 afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset, uint32_t flags);
+	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name);
+	      loc_t *loc, const char *name, dict_t *xdata);
 
 int32_t
 afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name);
+               fd_t *fd, const char *name, dict_t *xdata);
 
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 1d25949db..3dacfc8dd 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -46,143 +37,303 @@
 
 #include "afr.h"
 #include "afr-transaction.h"
-#include "afr-self-heal-common.h"
+//#include "afr-self-heal-common.h"
 
-/* {{{ writev */
 
-int
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int read_subvol = 0;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (local->inode) {
+		if (local->transaction.type == AFR_METADATA_TRANSACTION)
+			read_subvol = afr_metadata_subvol_get (local->inode, this,
+							       NULL, NULL);
+		else
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   NULL, NULL);
+	}
+
+	local->op_ret = -1;
+	local->op_errno = afr_final_errno (local, priv);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret < 0) {
+			afr_inode_read_subvol_reset (local->inode, this);
+			continue;
+		}
+
+		/* Order of checks in the compound conditional
+		   below is important.
+
+		   - Highest precedence: largest op_ret
+		   - Next precendence: if all op_rets are equal, read subvol
+		   - Least precedence: any succeeded subvol
+		*/
+		if ((local->op_ret < local->replies[i].op_ret) ||
+		    ((local->op_ret == local->replies[i].op_ret) &&
+		     (i == read_subvol))) {
+
+			local->op_ret = local->replies[i].op_ret;
+			local->op_errno = local->replies[i].op_errno;
+
+			local->cont.inode_wfop.prebuf =
+				local->replies[i].prestat;
+			local->cont.inode_wfop.postbuf =
+				local->replies[i].poststat;
+
+			if (local->replies[i].xdata) {
+				if (local->xdata_rsp)
+					dict_unref (local->xdata_rsp);
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			}
+		}
+	}
+}
+
+
+static void
+__afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
+			int op_ret, int op_errno,
+			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+
+	if (op_ret >= 0) {
+		if (prebuf)
+			local->replies[child_index].prestat = *prebuf;
+		if (postbuf)
+			local->replies[child_index].poststat = *postbuf;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	} else {
+		afr_transaction_fop_failed (frame, this, child_index);
+	}
+
+        return;
+}
+
+
+static int
+__afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
 
         local = frame->local;
 
         LOCK (&frame->lock);
         {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
+                __afr_inode_write_fill (frame, this, child_index, op_ret,
+					op_errno, prebuf, postbuf, xdata);
         }
         UNLOCK (&frame->lock);
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (writev, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.writev.prebuf,
-                                  &local->cont.writev.postbuf);
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+		__afr_inode_write_finalize (frame, this);
+
+		if (afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
+
+                local->transaction.resume (frame, this);
         }
+
         return 0;
 }
 
+/* {{{ writev */
 
-int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+void
+afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
 {
-        afr_local_t *   local = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int read_child  = 0;
+        afr_local_t *src_local = NULL;
+        afr_local_t *dst_local = NULL;
+
+        src_local = src_frame->local;
+        dst_local = dst_frame->local;
+
+        dst_local->op_ret = src_local->op_ret;
+        dst_local->op_errno = src_local->op_errno;
+        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+	if (src_local->xdata_rsp)
+		dst_local->xdata_rsp = dict_ref (src_local->xdata_rsp);
+}
 
+void
+afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
         local = frame->local;
 
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+        AFR_STACK_UNWIND (writev, frame,
+                          local->op_ret, local->op_errno,
+                          &local->cont.inode_wfop.prebuf,
+                          &local->cont.inode_wfop.postbuf,
+                          local->xdata_rsp);
+}
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret              = op_ret;
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
+int
+afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t *fop_frame = NULL;
 
-                        if (child_index == read_child) {
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
-                }
+        fop_frame = afr_transaction_detach_fop_frame (frame);
 
-                local->op_errno = op_errno;
+        if (fop_frame) {
+                afr_writev_copy_outvars (frame, fop_frame);
+                afr_writev_unwind (fop_frame, this);
         }
-        UNLOCK (&frame->lock);
+        return 0;
+}
 
-        call_count = afr_frame_return (frame);
+static void
+afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int           i      = 0;
 
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
+        local = frame->local;
+        priv = this->private;
+        /*
+         * We already have the best case result of the writev calls staged
+         * as the return value. Any writev that returns some value less
+         * than the best case is now out of sync, so mark the fop as
+         * failed. Note that fops that have returned with errors have
+         * already been marked as failed.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if ((!local->replies[i].valid) ||
+                    (local->replies[i].op_ret == -1))
+                        continue;
 
-                local->transaction.resume (frame, this);
+                if (local->replies[i].op_ret < local->op_ret)
+                        afr_transaction_fop_failed(frame, this, i);
         }
-        return 0;
 }
 
 int
-afr_writev_wind (call_frame_t *frame, xlator_t *this)
+afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int call_count = -1;
+        afr_local_t *   local = NULL;
+        call_frame_t    *fop_frame = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int ret = 0;
+        uint32_t open_fd_count = 0;
+        uint32_t write_is_append = 0;
 
         local = frame->local;
-        priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
+        LOCK (&frame->lock);
+        {
+                __afr_inode_write_fill (frame, this, child_index, op_ret,
+					op_errno, prebuf, postbuf, xdata);
+		if (op_ret == -1 || !xdata)
+			goto unlock;
+
+		write_is_append = 0;
+		ret = dict_get_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
+				       &write_is_append);
+		if (ret || !write_is_append)
+			local->append_write = _gf_false;
+
+		ret = dict_get_uint32 (xdata, GLUSTERFS_OPEN_FD_COUNT,
+				       &open_fd_count);
+		if (ret == -1)
+			goto unlock;
+		if ((open_fd_count > local->open_fd_count)) {
+			local->open_fd_count = open_fd_count;
+			local->update_open_fd_count = _gf_true;
+		}
         }
+unlock:
+        UNLOCK (&frame->lock);
 
-        local->call_count = call_count;
+        call_count = afr_frame_return (frame);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_writev_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->writev,
-                                           local->fd,
-                                           local->cont.writev.vector,
-                                           local->cont.writev.count,
-                                           local->cont.writev.offset,
-                                           local->cont.writev.flags,
-                                           local->cont.writev.iobref);
-
-                        if (!--call_count)
-                                break;
+        if (call_count == 0) {
+		if (!local->stable_write && !local->append_write)
+			/* An appended write removes the necessity to
+			   fsync() the file. This is because self-heal
+			   has the logic to check for larger file when
+			   the xattrs are not reliably pointing at
+			   a stale file.
+			*/
+			afr_fd_report_unstable_write (this, local->fd);
+
+		__afr_inode_write_finalize (frame, this);
+
+                afr_writev_handle_short_writes (frame, this);
+
+                if (local->update_open_fd_count)
+                        afr_handle_open_fd_count (frame, this);
+
+                if (!afr_txn_nothing_failed (frame, this)) {
+                        //Don't unwind until post-op is complete
+                        local->transaction.resume (frame, this);
+                } else {
+                /*
+                 * Generally inode-write fops do transaction.unwind then
+                 * transaction.resume, but writev needs to make sure that
+                 * delayed post-op frame is placed in fdctx before unwind
+                 * happens. This prevents the race of flush doing the
+                 * changelog wakeup first in fuse thread and then this
+                 * writev placing its delayed post-op frame in fdctx.
+                 * This helps flush make sure all the delayed post-ops are
+                 * completed.
+                 */
+
+                        fop_frame = afr_transaction_detach_fop_frame (frame);
+                        afr_writev_copy_outvars (frame, fop_frame);
+                        local->transaction.resume (frame, this);
+                        afr_writev_unwind (fop_frame, this);
                 }
         }
-
         return 0;
 }
 
 
 int
-afr_writev_done (call_frame_t *frame, xlator_t *this)
+afr_writev_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
         local = frame->local;
+        priv = this->private;
 
-        iobref_unref (local->cont.writev.iobref);
-        local->cont.writev.iobref = NULL;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->writev,
+			   local->fd, local->cont.writev.vector,
+			   local->cont.writev.count, local->cont.writev.offset,
+			   local->cont.writev.flags, local->cont.writev.iobref,
+			   local->xdata_req);
         return 0;
 }
 
@@ -192,30 +343,37 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t    *transaction_frame = NULL;
         afr_local_t     *local             = NULL;
-        int             op_ret   = -1;
-        int             op_errno = 0;
-
-        local = frame->local;
+        int             ret   = -1;
+        int             op_errno = ENOMEM;
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
+        local = frame->local;
         transaction_frame->local = local;
-        frame->local = NULL;
+	frame->local = NULL;
 
-        local->op = GF_FOP_WRITE;
+	if (!AFR_FRAME_INIT (frame, op_errno))
+		goto out;
 
-        local->success_count      = 0;
+        local->op = GF_FOP_WRITE;
 
-        local->transaction.fop    = afr_writev_wind;
-        local->transaction.done   = afr_writev_done;
-        local->transaction.unwind = afr_writev_unwind;
+        local->transaction.wind   = afr_writev_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_transaction_writev_unwind;
 
         local->transaction.main_frame = frame;
+
         if (local->fd->flags & O_APPEND) {
+               /*
+                * Backend vfs ignores the 'offset' for append mode fd so
+                * locking just the region provided for the writev does not
+                * give consistency gurantee. The actual write may happen at a
+                * completely different range than the one provided by the
+                * offset, len in the fop. So lock the entire file.
+                */
                 local->transaction.start   = 0;
                 local->transaction.len     = 0;
         } else {
@@ -224,265 +382,86 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
                                                          local->cont.writev.count);
         }
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
-        }
-
-        return 0;
-}
-
-static int
-afr_prepare_loc (call_frame_t *frame, fd_t *fd)
-{
-        afr_local_t    *local = NULL;
-        char           *name = NULL;
-        char           *path = NULL;
-        int             ret = 0;
-
-        if ((!fd) || (!fd->inode))
-                return -1;
-
-        local = frame->local;
-        ret = inode_path (fd->inode, NULL, (char **)&path);
-        if (ret <= 0) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "Unable to get path for gfid: %s",
-                        uuid_utoa (fd->inode->gfid));
-                return -1;
-        }
-
-        if (local->loc.path) {
-                if (strcmp (path, local->loc.path))
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "overwriting old loc->path %s with %s",
-                                local->loc.path, path);
-                GF_FREE ((char *)local->loc.path);
-        }
-        local->loc.path = path;
-
-        name = strrchr (local->loc.path, '/');
-        if (name)
-                name++;
-        local->loc.name = name;
-
-        if (local->loc.inode) {
-                inode_unref (local->loc.inode);
-        }
-        local->loc.inode = inode_ref (fd->inode);
-
-        if (local->loc.parent) {
-                inode_unref (local->loc.parent);
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
         }
 
-        local->loc.parent = inode_parent (local->loc.inode, 0, NULL);
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
+	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
-afr_fd_paused_call_t*
-afr_paused_call_create (call_frame_t *frame)
-{
-        afr_local_t             *local = NULL;
-        afr_fd_paused_call_t   *paused_call = NULL;
-
-        local = frame->local;
-        GF_ASSERT (local->fop_call_continue);
-
-        paused_call = GF_CALLOC (1, sizeof (*paused_call),
-                                  gf_afr_fd_paused_call_t);
-        if (paused_call) {
-                INIT_LIST_HEAD (&paused_call->call_list);
-                paused_call->frame = frame;
-        }
-
-        return paused_call;
-}
-
-static int
-afr_pause_fd_fop (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        int                    ret = 0;
-
-        paused_call = afr_paused_call_create (frame);
-        if (paused_call)
-                list_add (&paused_call->call_list, &fd_ctx->paused_calls);
-        else
-                ret = -ENOMEM;
-
-        return ret;
-}
-
-static void
-afr_trigger_open_fd_self_heal (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t             *local = NULL;
-        afr_self_heal_t         *sh = NULL;
-        inode_t                 *inode = NULL;
-        char                    *reason = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        inode = local->fd->inode;
-
-        sh->do_missing_entry_self_heal = _gf_true;
-        sh->do_gfid_self_heal = _gf_true;
-        sh->do_data_self_heal = _gf_true;
-
-        reason = "subvolume came online";
-        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
-                              reason, NULL, NULL);
-}
-
-int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop)
-{
-        int                     ret = 0;
-        int                     i   = 0;
-        afr_fd_ctx_t            *fd_ctx = NULL;
-        gf_boolean_t            need_self_heal = _gf_false;
-        int                     *need_open = NULL;
-        int                     need_open_count = 0;
-        afr_local_t             *local = NULL;
-        afr_private_t           *priv = NULL;
-        gf_boolean_t            fop_continue = _gf_true;
-
-        local = frame->local;
-        priv  = this->private;
-
-        GF_ASSERT (local->fd);
-
-        if (fd_is_anonymous (local->fd)) {
-                fop_continue = _gf_true;
-                goto out;
-        }
-
-        if (pause_fop)
-                GF_ASSERT (local->fop_call_continue);
-
-        ret = afr_prepare_loc (frame, local->fd);
-        if (ret < 0) {
-                //File does not exist we cant open it.
-                ret = 0;
-                goto out;
-        }
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-        if (!fd_ctx) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        LOCK (&local->fd->lock);
-        {
-                if (fd_ctx->up_count < priv->up_count) {
-                        need_self_heal = _gf_true;
-                        fd_ctx->up_count   = priv->up_count;
-                        fd_ctx->down_count = priv->down_count;
-                }
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if ((fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED) &&
-                            local->child_up[i]) {
-                                fd_ctx->opened_on[i] = AFR_FD_OPENING;
-                                if (!need_open)
-                                        need_open = GF_CALLOC (priv->child_count,
-                                                               sizeof (*need_open),
-                                                               gf_afr_mt_int32_t);
-                                need_open[i] = 1;
-                                need_open_count++;
-                        } else if (pause_fop && local->child_up[i] &&
-                                   (fd_ctx->opened_on[i] == AFR_FD_OPENING)) {
-                                local->fop_paused = _gf_true;
-                        }
-                }
-
-                if (local->fop_paused) {
-                        GF_ASSERT (pause_fop);
-                        gf_log (this->name, GF_LOG_INFO, "Pause fd %p",
-                                local->fd);
-                        ret = afr_pause_fd_fop (frame, this, fd_ctx);
-                        if (ret)
-                                goto unlock;
-                        fop_continue = _gf_false;
-                }
-        }
-unlock:
-        UNLOCK (&local->fd->lock);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to fix fd for %s",
-                        local->loc.path);
-                fop_continue = _gf_false;
-                goto out;
-        }
-
-        if (need_self_heal)
-                afr_trigger_open_fd_self_heal (frame, this);
-
-        if (!need_open_count)
-                goto out;
-
-        gf_log (this->name, GF_LOG_INFO, "Opening fd %p", local->fd);
-        afr_fix_open (frame, this, fd_ctx, need_open_count, need_open);
-        fop_continue = _gf_false;
-out:
-        if (need_open)
-                GF_FREE (need_open);
-        if (fop_continue && local->fop_call_continue)
-                local->fop_call_continue (frame, this);
-        return ret;
-}
 
 int
 afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref)
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(writev,out);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->cont.writev.vector     = iov_dup (vector, count);
+        local->cont.writev.vector = iov_dup (vector, count);
+	if (!local->cont.writev.vector)
+		goto out;
         local->cont.writev.count      = count;
         local->cont.writev.offset     = offset;
         local->cont.writev.flags      = flags;
         local->cont.writev.iobref     = iobref_ref (iobref);
 
-        local->fd                = fd_ref (fd);
-        local->fop_call_continue = afr_do_writev;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+	if (!local->xdata_req)
+		goto out;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_OPEN_FD_COUNT, 4)) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
-        ret = 0;
+	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
+		op_errno = ENOMEM;
+		goto out;
+	}
+
+	/* Set append_write to be true speculatively. If on any
+	   server it turns not be true, we unset it in the
+	   callback.
+	*/
+	local->append_write = _gf_true;
+
+	/* detect here, but set it in writev_wind_cbk *after* the unstable
+	   write is performed
+	*/
+	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
+
+        afr_fix_open (fd, this);
+
+        afr_do_writev (frame, this);
+
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -500,21 +479,13 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.truncate.prebuf,
-                                  &local->cont.truncate.postbuf);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (truncate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
@@ -522,172 +493,99 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf)
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
+	afr_local_t *local = NULL;
 
-        read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
+	local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != EFBIG)
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+		local->stable_write = _gf_false;
 
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
 }
 
 
-int32_t
-afr_truncate_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_truncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->truncate,
-                                           &local->loc,
-                                           local->cont.truncate.offset);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_truncate_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->truncate,
+			   &local->loc, local->cont.truncate.offset,
+			   local->xdata_req);
         return 0;
 }
 
 
 int
 afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset)
+              loc_t *loc, off_t offset, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
         call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	int ret = -1;
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(truncate,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.truncate.offset  = offset;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_truncate_wind;
-        local->transaction.done   = afr_truncate_done;
+	if (!local->xdata_req)
+		goto out;
+
+	local->transaction.wind   = afr_truncate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_truncate_unwind;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+        local->op = GF_FOP_TRUNCATE;
 
         local->transaction.main_frame = frame;
         local->transaction.start   = offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+	/* Set it true speculatively, will get reset in afr_truncate_wind_cbk
+	   if truncate was not a NOP */
+	local->stable_write = _gf_true;
 
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -705,20 +603,13 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.ftruncate.prebuf,
-                                  &local->cont.ftruncate.postbuf);
-        }
+	AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
@@ -726,140 +617,77 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int need_unwind = 0;
-        int read_child  = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
+	afr_local_t *local = NULL;
 
-                        if (child_index == read_child) {
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
+	local = frame->local;
 
-                        local->success_count++;
+	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+		local->stable_write = _gf_false;
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
 }
 
 
 int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
 
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->ftruncate,
-                                           local->fd, local->cont.ftruncate.offset);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+	local = frame->local;
+	priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->ftruncate,
+			   local->fd, local->cont.ftruncate.offset,
+			   local->xdata_req);
         return 0;
 }
 
 
 int
-afr_ftruncate_done (call_frame_t *frame, xlator_t *this)
+afr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       dict_t *xdata)
 {
+        afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
+	call_frame_t *transaction_frame = NULL;
+	int ret = -1;
+        int op_errno = ENOMEM;
 
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+        priv = this->private;
 
-        return 0;
-}
+        QUORUM_CHECK(ftruncate,out);
 
+	transaction_frame = copy_frame (frame);
+	if (!frame)
+		goto out;
 
-int
-afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
-{
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
-        int op_ret   = -1;
-        int op_errno = 0;
+        local = AFR_FRAME_INIT (transaction_frame, op_errno);
+        if (!local)
+		goto out;
 
-        local = frame->local;
+        local->cont.ftruncate.offset  = offset;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
+	if (!local->xdata_req)
+		goto out;
 
-        transaction_frame->local = local;
-        frame->local = NULL;
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
         local->op = GF_FOP_FTRUNCATE;
 
-        local->transaction.fop    = afr_ftruncate_wind;
-        local->transaction.done   = afr_ftruncate_done;
+	local->transaction.wind   = afr_ftruncate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_ftruncate_unwind;
 
         local->transaction.main_frame = frame;
@@ -867,63 +695,21 @@ afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
         local->transaction.start   = local->cont.ftruncate.offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        afr_fix_open (fd, this);
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL, NULL);
-        }
-
-        return 0;
-}
-
-
-int
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, off_t offset)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+	/* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
+	   if truncate was not a NOP */
+	local->stable_write = _gf_true;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(ftruncate,out);
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->cont.ftruncate.offset  = offset;
-
-        local->fd = fd_ref (fd);
-        local->fop_call_continue = afr_do_ftruncate;
-
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
-        }
+	AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -935,201 +721,109 @@ out:
 int
 afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *main_frame = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.setattr.preop_buf,
-                                  &local->cont.setattr.postop_buf);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (setattr, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
 afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+                      int op_ret, int op_errno,
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      preop, postop, xdata);
 }
 
 
-int32_t
-afr_setattr_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_setattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setattr,
-                                           &local->loc,
-                                           &local->cont.setattr.in_buf,
-                                           local->cont.setattr.valid);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->setattr,
+			   &local->loc, &local->cont.setattr.in_buf,
+			   local->cont.setattr.valid, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_setattr_done (call_frame_t *frame, xlator_t *this)
+afr_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+	     int32_t valid, dict_t *xdata)
 {
+        afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+        call_frame_t *transaction_frame = NULL;
         int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(setattr,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.setattr.in_buf = *buf;
         local->cont.setattr.valid  = valid;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_setattr_wind;
-        local->transaction.done   = afr_setattr_done;
+        local->transaction.wind   = afr_setattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_setattr_unwind;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+	local->op = GF_FOP_SETATTR;
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1143,21 +837,13 @@ afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.fsetattr.preop_buf,
-                                  &local->cont.fsetattr.postop_buf);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
@@ -1165,179 +851,91 @@ afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop)
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      preop, postop, xdata);
 }
 
 
-int32_t
-afr_fsetattr_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_fsetattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsetattr,
-                                           local->fd,
-                                           &local->cont.fsetattr.in_buf,
-                                           local->cont.fsetattr.valid);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fsetattr,
+			   local->fd, &local->cont.fsetattr.in_buf,
+			   local->cont.fsetattr.valid, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_fsetattr_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-int
 afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid)
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
         int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(fsetattr,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.fsetattr.in_buf = *buf;
         local->cont.fsetattr.valid  = valid;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_fsetattr_wind;
-        local->transaction.done   = afr_fsetattr_done;
+        local->transaction.wind   = afr_fsetattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_fsetattr_unwind;
 
         local->fd                 = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+	local->op = GF_FOP_FSETATTR;
+
+        afr_fix_open (fd, this);
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1353,177 +951,108 @@ afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno);
-        }
+	AFR_STACK_UNWIND (setxattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
 afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               need_unwind   = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
 }
 
 
 int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
+afr_setxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t      *local         = NULL;
         afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               i             = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setxattr,
-                                           &local->loc,
-                                           local->cont.setxattr.dict,
-                                           local->cont.setxattr.flags);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->setxattr,
+			   &local->loc, local->cont.setxattr.dict,
+			   local->cont.setxattr.flags, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local    = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-int
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags)
+afr_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+	      int32_t flags, dict_t *xdata)
 {
         afr_private_t  *priv              = NULL;
         afr_local_t    *local             = NULL;
         call_frame_t   *transaction_frame = NULL;
-        data_pair_t    *trav              = NULL;
         int             ret               = -1;
         int             op_errno          = EINVAL;
 
-        VALIDATE_OR_GOTO (this, out);
-
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
+                                   op_errno, out);
 
         priv = this->private;
 
         QUORUM_CHECK(setxattr,out);
+
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
                 goto out;
 
         local->cont.setxattr.dict  = dict_ref (dict);
         local->cont.setxattr.flags = flags;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_setxattr_wind;
-        local->transaction.done   = afr_setxattr_done;
+        local->transaction.wind   = afr_setxattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_setxattr_unwind;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+	local->op = GF_FOP_SETXATTR;
 
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
@@ -1538,182 +1067,108 @@ afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (fsetxattr, main_frame,
-                                  local->op_ret, local->op_errno);
-        }
+	AFR_STACK_UNWIND (fsetxattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
 afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t     *local       = NULL;
-        afr_private_t   *priv        = NULL;
-        int              call_count  = -1;
-        int              need_unwind = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
 }
 
 
 int
-afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this)
+afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t        *local       = NULL;
         afr_private_t      *priv        = NULL;
-        int                 call_count  = -1;
-        int                 i           = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsetxattr,
-                                           local->fd,
-                                           local->cont.fsetxattr.dict,
-                                           local->cont.fsetxattr.flags);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fsetxattr,
+			   local->fd, local->cont.fsetxattr.dict,
+			   local->cont.fsetxattr.flags, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_fsetxattr_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local   = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-int
 afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags)
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
 {
         afr_private_t    *priv              = NULL;
         afr_local_t      *local             = NULL;
         call_frame_t     *transaction_frame = NULL;
         int               ret               = -1;
-        int               op_errno          = EINVAL;
-        data_pair_t      *trav              = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int               op_errno          = ENOMEM;
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
-
-        if (ret)
-                goto out;
+                                   op_errno, out);
 
         priv = this->private;
 
         QUORUM_CHECK(fsetxattr,out);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
+        if (!transaction_frame)
                 goto out;
-        }
 
-        transaction_frame->local = local;
-
-        local->op_ret = -1;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.fsetxattr.dict  = dict_ref (dict);
         local->cont.fsetxattr.flags = flags;
 
-        local->transaction.fop    = afr_fsetxattr_wind;
-        local->transaction.done   = afr_fsetxattr_done;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_fsetxattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_fsetxattr_unwind;
 
         local->fd                 = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	local->op = GF_FOP_FSETXATTR;
 
         local->transaction.main_frame = frame;
         local->transaction.start  = LLONG_MAX - 1;
         local->transaction.len    = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -1731,127 +1186,165 @@ afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (removexattr, main_frame,
-                                  local->op_ret, local->op_errno);
-        }
+	AFR_STACK_UNWIND (removexattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
 afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
+}
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
 
-        call_count = afr_frame_return (frame);
+int
+afr_removexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	local = frame->local;
+	priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->removexattr,
+			   &local->loc, local->cont.removexattr.name,
+			   local->xdata_req);
         return 0;
 }
 
 
-int32_t
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_removexattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, const char *name, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+        afr_private_t   *priv              = NULL;
+        afr_local_t     *local             = NULL;
+        call_frame_t    *transaction_frame = NULL;
+        int              ret               = -1;
+        int              op_errno          = ENOMEM;
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
+                                 name, op_errno, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
+                                 name, op_errno, out);
 
-        local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+        QUORUM_CHECK(removexattr,out);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-        local->call_count = call_count;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->removexattr,
-                                           &local->loc,
-                                           local->cont.removexattr.name);
-
-                        if (!--call_count)
-                                break;
-                }
+        local->cont.removexattr.name = gf_strdup (name);
+
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_removexattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_removexattr_unwind;
+
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+	local->op = GF_FOP_REMOVEXATTR;
+
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         return 0;
 }
 
-
+/* ffremovexattr */
 int
-afr_removexattr_done (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = frame->local;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
 
-        local->transaction.unwind (frame, this);
+        local = frame->local;
 
-        AFR_STACK_DESTROY (frame);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (fremovexattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name)
+afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_private_t   *priv              = NULL;
-        afr_local_t     *local             = NULL;
-        call_frame_t    *transaction_frame = NULL;
-        int              ret               = -1;
-        int              op_errno          = 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
+}
+
+
+int
+afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-        VALIDATE_OR_GOTO (this, out);
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fremovexattr,
+			   local->fd, local->cont.removexattr.name,
+			   local->xdata_req);
+        return 0;
+}
+
+
+int
+afr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  const char *name, dict_t *xdata)
+{
+	afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
         GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
                                  name, op_errno, out);
@@ -1859,235 +1352,403 @@ afr_removexattr (call_frame_t *frame, xlator_t *this,
         GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
                                  name, op_errno, out);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+	priv = this->private;
 
-        priv = this->private;
-
-        QUORUM_CHECK(removexattr,out);
+        QUORUM_CHECK(fremovexattr, out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
                 goto out;
 
         local->cont.removexattr.name = gf_strdup (name);
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_removexattr_wind;
-        local->transaction.done   = afr_removexattr_done;
-        local->transaction.unwind = afr_removexattr_unwind;
+	if (!local->xdata_req)
+		goto out;
 
-        loc_copy (&local->loc, loc);
+        local->transaction.wind   = afr_fremovexattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fremovexattr_unwind;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	local->op = GF_FOP_FREMOVEXATTR;
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
-/* ffremovexattr */
+
 int
-afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *   local = NULL;
         call_frame_t   *main_frame = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (fremovexattr, main_frame,
-                                  local->op_ret, local->op_errno);
-        }
+	AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
+
+
+int
+afr_fallocate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
         local = frame->local;
         priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
+	STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fallocate,
+			   local->fd, local->cont.fallocate.mode,
+			   local->cont.fallocate.offset,
+			   local->cont.fallocate.len, local->xdata_req);
+        return 0;
+}
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata)
+{
+	afr_private_t *priv = NULL;
+        call_frame_t *transaction_frame = NULL;
+        afr_local_t *local = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
-        call_count = afr_frame_return (frame);
+	priv = this->private;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
+        QUORUM_CHECK(fallocate,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
+
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->cont.fallocate.mode = mode;
+        local->cont.fallocate.offset  = offset;
+        local->cont.fallocate.len = len;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_FALLOCATE;
+
+        local->transaction.wind   = afr_fallocate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fallocate_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.fallocate.offset;
+        local->transaction.len     = 0;
+
+        afr_fix_open (fd, this);
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+/* }}} */
+
+/* {{{ discard */
+
+int
+afr_discard_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
+
+	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
 
-int32_t
-afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
+
+
+int
+afr_discard_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+	STACK_WIND_COOKIE (frame, afr_discard_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->discard,
+			   local->fd, local->cont.discard.offset,
+			   local->cont.discard.len, local->xdata_req);
+        return 0;
+}
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
 
-        local->call_count = call_count;
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
+
+        priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fremovexattr,
-                                           local->fd,
-                                           local->cont.removexattr.name);
-
-                        if (!--call_count)
-                                break;
-                }
+        QUORUM_CHECK(discard, out);
+
+	transaction_frame = copy_frame (frame);
+	if (!transaction_frame)
+		goto out;
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->cont.discard.offset  = offset;
+        local->cont.discard.len = len;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_DISCARD;
+
+        local->transaction.wind   = afr_discard_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_discard_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = 0;
+
+        afr_fix_open (fd, this);
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
+/* {{{ zerofill */
+
 int
-afr_fremovexattr_done (call_frame_t *frame, xlator_t *this)
+afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = frame->local;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
 
-        local->transaction.unwind (frame, this);
+        local = frame->local;
 
-        AFR_STACK_DESTROY (frame);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name)
+afr_zerofill_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_ret   = -1;
-        int op_errno = 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
 
-        VALIDATE_OR_GOTO (this, out);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+int
+afr_zerofill_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->zerofill,
+			   local->fd, local->cont.zerofill.offset,
+			   local->cont.zerofill.len, local->xdata_req);
+        return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
+int
+afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
-        QUORUM_CHECK(fremovexattr, out);
+        QUORUM_CHECK(discard, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
+	transaction_frame = copy_frame (frame);
+	if (!transaction_frame)
+		goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        local->cont.zerofill.offset  = offset;
+        local->cont.zerofill.len = len;
 
-        transaction_frame->local = local;
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        local->op_ret = -1;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->cont.removexattr.name = gf_strdup (name);
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_fremovexattr_wind;
-        local->transaction.done   = afr_fremovexattr_done;
-        local->transaction.unwind = afr_fremovexattr_unwind;
+        local->op = GF_FOP_ZEROFILL;
 
-        local->fd = fd_ref (fd);
+        local->transaction.wind   = afr_zerofill_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_zerofill_unwind;
 
         local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = len;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno);
+        afr_fix_open (fd, this);
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
+
+/* }}} */
+
+
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 729a490d5..7b1fc5528 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_WRITE_H__
@@ -22,59 +13,70 @@
 
 int32_t
 afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode);
+	   loc_t *loc, mode_t mode, dict_t *xdata);
 
 int32_t
 afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid);
+	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
 
 int
 afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid);
+	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
 
 int32_t
 afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode);
+	    fd_t *fd, mode_t mode, dict_t *xdata);
 
 int32_t
 afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	    struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref);
+            uint32_t flags, struct iobref *iobref, dict_t *xdata);
 
 int32_t
 afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset);
+	      loc_t *loc, off_t offset, dict_t *xdata);
 
 int32_t
 afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset);
+	       fd_t *fd, off_t offset, dict_t *xdata);
 
 int32_t
 afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2]);
+	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
 
 int
 afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid);
+             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int
 afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid);
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int32_t
 afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags);
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
 afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags);
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
 afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name);
+		 loc_t *loc, const char *name, dict_t *xdata);
 
 int32_t
 afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name);
+                  fd_t *fd, const char *name, dict_t *xdata);
+
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
 
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata);
+
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index 39646ff85..a2a758f35 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
@@ -31,8 +22,69 @@
 #define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
 #define LOCKED_LOWER    0x2        /* for lower path */
 
+#define AFR_TRACE_INODELK_IN(frame, this, params ...)           \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->inodelk_trace)                      \
+                        break;                                  \
+                afr_trace_inodelk_in (frame, this, params);     \
+        } while (0);
+
+#define AFR_TRACE_INODELK_OUT(frame, this, params ...)          \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->inodelk_trace)                      \
+                        break;                                  \
+                afr_trace_inodelk_out (frame, this, params);    \
+        } while (0);
+
+#define AFR_TRACE_ENTRYLK_IN(frame, this, params ...)           \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->entrylk_trace)                      \
+                        break;                                  \
+                afr_trace_entrylk_in (frame, this, params);     \
+        } while (0);
+
+#define AFR_TRACE_ENTRYLK_OUT(frame, this, params ...)          \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->entrylk_trace)                      \
+                        break;                                  \
+                afr_trace_entrylk_out (frame, this, params);    \
+        } while (0);
+
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+afr_entry_lockee_cmp (const void *l1, const void *l2)
+{
+        const afr_entry_lockee_t       *r1 = l1;
+        const afr_entry_lockee_t       *r2 = l2;
+        int                            ret = 0;
+        uuid_t                         gfid1 = {0};
+        uuid_t                         gfid2 = {0};
+
+        loc_gfid ((loc_t*)&r1->loc, gfid1);
+        loc_gfid ((loc_t*)&r2->loc, gfid2);
+        ret = uuid_compare (gfid1, gfid2);
+        /*Entrylks with NULL basename are the 'smallest'*/
+        if (ret == 0) {
+                if (!r1->basename)
+                        return -1;
+                if (!r2->basename)
+                        return 1;
+                ret = strcmp (r1->basename, r2->basename);
+        }
+
+        if (ret <= 0)
+                return -1;
+        else
+                return 1;
+}
+
+int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+
+static int
+afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
 
 static uint64_t afr_lock_number = 1;
 
@@ -57,13 +109,13 @@ afr_set_lock_number (call_frame_t *frame, xlator_t *this)
 }
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this)
+afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner)
 {
         gf_log (this->name, GF_LOG_TRACE,
                 "Setting lk-owner=%llu",
-                (unsigned long long) (unsigned long)frame->root);
+                (unsigned long long) (unsigned long)lk_owner);
 
-        set_lk_owner_from_ptr (&frame->root->lk_owner, frame->root);
+        set_lk_owner_from_ptr (&frame->root->lk_owner, lk_owner);
 }
 
 static int
@@ -99,16 +151,9 @@ internal_lock_count (call_frame_t *frame, xlator_t *this)
         local = frame->local;
         priv  = this->private;
 
-        if (local->fd) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i])
-                                ++call_count;
-                }
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i])
-                                ++call_count;
-                }
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i])
+                        ++call_count;
         }
 
         return call_count;
@@ -242,27 +287,20 @@ afr_set_lock_call_type (afr_lock_call_type_t lock_call_type,
 }
 
 static void
-afr_trace_inodelk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_inodelk_out (call_frame_t *frame, xlator_t *this,
+                       afr_lock_call_type_t lock_call_type,
                        afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
                        int op_ret, int op_errno, int32_t child_index)
 {
-        xlator_t            *this     = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
 
         char lockee[256];
         char lock_call_type_str[256];
         char verdict[16];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->inodelk_trace) {
-                return;
-        }
 
         afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
 
@@ -271,37 +309,29 @@ afr_trace_inodelk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
         afr_print_verdict (op_ret, op_errno, verdict);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lockee={%s} Number={%llu}",
+                "[%s %s] [%s] lk-owner=%s Lockee={%s} Number={%llu}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lockee,
+                verdict, lkowner_utoa (&frame->root->lk_owner), lockee,
                 (unsigned long long) int_lock->lock_number);
 
 }
 
 static void
-afr_trace_inodelk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_inodelk_in (call_frame_t *frame, xlator_t *this,
+                      afr_lock_call_type_t lock_call_type,
                       afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
                       int32_t cmd, int32_t child_index)
 {
-        xlator_t            *this     = NULL;
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->inodelk_trace) {
-                return;
-        }
 
         afr_print_inodelk (lock, 256, cmd, flock, &frame->root->lk_owner);
         afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
@@ -318,20 +348,21 @@ afr_trace_inodelk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
 }
 
 static void
-afr_trace_entrylk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
+                      afr_lock_call_type_t lock_call_type,
                       afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t child_index)
+                      int32_t cookie)
 {
-        xlator_t            *this     = NULL;
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
+        int                 child_index = 0;
+        int                 lockee_no = 0;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
@@ -339,36 +370,41 @@ afr_trace_entrylk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
         if (!priv->entrylk_trace) {
                 return;
         }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
 
         afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
+                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}, Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 }
 
 static void
-afr_trace_entrylk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, const char *basename, int op_ret,
-                       int op_errno, int32_t child_index)
+afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
+                       afr_lock_call_type_t lock_call_type,
+                       afr_lock_op_type_t lk_op_type, const char *basename,
+                       int op_ret, int op_errno, int32_t cookie)
 {
-        xlator_t            *this     = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
+        int                 lockee_no = 0;
+        int                 child_index = 0;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
         char verdict[16];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
@@ -376,20 +412,25 @@ afr_trace_entrylk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
         if (!priv->entrylk_trace) {
                 return;
         }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
 
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         afr_print_verdict (op_ret, op_errno, verdict);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu}",
+                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu} Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
                 verdict,
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 
 }
 
@@ -442,6 +483,47 @@ is_afr_lock_transaction (afr_local_t *local)
         return ret;
 }
 
+int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count)
+{
+        int     ret     = -1;
+
+        loc_copy (&lockee->loc, loc);
+        lockee->basename        = (basename)? gf_strdup (basename): NULL;
+        if (basename && !lockee->basename)
+                goto out;
+
+        lockee->locked_count    = 0;
+        lockee->locked_nodes    = GF_CALLOC (child_count,
+                                             sizeof (*lockee->locked_nodes),
+                                             gf_afr_mt_afr_node_character);
+
+        if (!lockee->locked_nodes)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
+{
+        int     i   = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++) {
+                loc_wipe (&int_lock->lockee[i].loc);
+                if (int_lock->lockee[i].basename)
+                        GF_FREE (int_lock->lockee[i].basename);
+                if (int_lock->lockee[i].locked_nodes)
+                       GF_FREE (int_lock->lockee[i].locked_nodes);
+        }
+
+        return;
+}
+
 static int
 initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
 {
@@ -459,8 +541,13 @@ initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
         int_lock->lock_op_ret        = -1;
         int_lock->lock_op_errno      = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->entry_locked_nodes[i] = 0;
+        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+                if (!int_lock->lockee[i].locked_nodes)
+                        break;
+                int_lock->lockee[i].locked_count = 0;
+                memset (int_lock->lockee[i].locked_nodes, 0,
+                        sizeof (*int_lock->lockee[i].locked_nodes) *
+                        priv->child_count);
         }
 
         return 0;
@@ -472,37 +559,37 @@ initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
-        int i = 0;
+        afr_inodelk_t       *inodelk  = NULL;
 
         priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        int_lock->inodelk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->inode_locked_nodes[i] = 0;
-        }
+        inodelk->lock_count    = 0;
+        int_lock->lk_attempted_count = 0;
+        int_lock->lock_op_ret   = -1;
+        int_lock->lock_op_errno = 0;
+
+        memset (inodelk->locked_nodes, 0,
+                sizeof (*inodelk->locked_nodes) * priv->child_count);
+        memset (int_lock->locked_nodes, 0,
+                sizeof (*int_lock->locked_nodes) * priv->child_count);
 
         return 0;
 }
 
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
+int
+afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
 {
-        int ret = 0;
-
-        ret = uuid_compare (l1->inode->gfid, l2->inode->gfid);
+        int call_count  = 0;
+        int i           = 0;
 
-        if (ret == 0)
-                ret = strcmp (b1, b2);
+        for (i = 0; i < int_lock->lockee_count; i++)
+                call_count += int_lock->lockee[i].locked_count;
 
-        if (ret <= 0)
-                return l1;
-        else
-                return l2;
+        return call_count;
 }
 
 int
@@ -523,7 +610,7 @@ afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
 /* FIXME: What if UNLOCK fails */
 static int32_t
 afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
@@ -549,33 +636,37 @@ afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 static int32_t
 afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local = NULL;
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk = NULL;
         int32_t             child_index = (long)cookie;
+        afr_private_t       *priv = NULL;
 
         local = frame->local;
         int_lock = &local->internal_lock;
 
-        afr_trace_inodelk_out (frame, AFR_INODELK_TRANSACTION,
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
                                AFR_UNLOCK_OP, NULL, op_ret,
                                op_errno, child_index);
 
+        priv = this->private;
+
         if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: unlock failed on %d, reason: %s",
-                        local->loc.path, child_index, strerror (op_errno));
+                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on subvolume %s "
+                        "with lock owner %s", local->loc.path,
+                        priv->children[child_index]->name,
+                        lkowner_utoa (&frame->root->lk_owner));
         }
 
 
-        int_lock->inode_locked_nodes[child_index] &= LOCKED_NO;
-
-        if (op_ret == 1) {
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->locked_nodes[child_index] &= LOCKED_NO;
+        if (local->transaction.eager_lock)
                 local->transaction.eager_lock[child_index] = 0;
-        }
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         return 0;
 
@@ -585,9 +676,12 @@ static int
 afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         struct gf_flock flock = {0,};
+        struct gf_flock full_flock = {0,};
+        struct gf_flock *flock_use = NULL;
         int call_count = 0;
         int i = 0;
         int piggyback = 0;
@@ -598,15 +692,14 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = F_UNLCK;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        gf_log (this->name, GF_LOG_DEBUG, "attempting data unlock range %"PRIu64
-                " %"PRIu64" by %s", flock.l_start, flock.l_len,
-                lkowner_utoa (&frame->root->lk_owner));
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = F_UNLCK;
 
-        call_count = afr_locked_nodes_count (int_lock->inode_locked_nodes,
+        full_flock.l_type = F_UNLCK;
+        call_count = afr_locked_nodes_count (inodelk->locked_nodes,
                                              priv->child_count);
 
         int_lock->lk_call_count = call_count;
@@ -622,11 +715,11 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                 fd_ctx = afr_fd_ctx_get (local->fd, this);
 
         for (i = 0; i < priv->child_count; i++) {
-                if ((int_lock->inode_locked_nodes[i] & LOCKED_YES)
-                    != LOCKED_YES)
+                if ((inodelk->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
                         continue;
 
                 if (local->fd) {
+                        flock_use = &flock;
                         if (!local->transaction.eager_lock[i]) {
                                 goto wind;
                         }
@@ -638,43 +731,48 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                                 if (fd_ctx->lock_piggyback[i]) {
                                         fd_ctx->lock_piggyback[i]--;
                                         piggyback = 1;
+                                } else {
+                                        fd_ctx->lock_acquired[i]--;
                                 }
                         }
                         UNLOCK (&local->fd->lock);
 
                         if (piggyback) {
                                 afr_unlock_inodelk_cbk (frame, (void *) (long) i,
-                                                        this, 1, 0);
+                                                        this, 1, 0, NULL);
                                 if (!--call_count)
                                         break;
                                 continue;
                         }
 
-                        fd_ctx->lock_acquired[i]--;
+                        flock_use = &full_flock;
                 wind:
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
-                                              AFR_UNLOCK_OP, &flock, F_SETLK, i);
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
+                                              AFR_UNLOCK_OP, flock_use, F_SETLK,
+                                              i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
                                            (void *) (long)i,
                                            priv->children[i],
                                            priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLK, &flock);
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
 
                         if (!--call_count)
                                 break;
 
                 } else {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
                                               AFR_UNLOCK_OP, &flock, F_SETLK, i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
                                            (void *) (long)i,
                                            priv->children[i],
                                            priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLK, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -686,24 +784,34 @@ out:
 
 static int32_t
 afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local = NULL;
-        int32_t             child_index = (long)cookie;
+        afr_private_t       *priv  = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int32_t             child_index = 0;
+        int                 lockee_no   = 0;
+
+        priv = this->private;
+        lockee_no = (int)((long) cookie) / priv->child_count;
+        child_index = (int) ((long) cookie) % priv->child_count;
 
         local = frame->local;
+        int_lock = &local->internal_lock;
 
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, child_index);
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_UNLOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
+                               op_errno, (int) ((long)cookie));
 
-        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        if (op_ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: unlock failed on %d, reason: %s",
                         local->loc.path, child_index, strerror (op_errno));
         }
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
 
         return 0;
 }
@@ -711,24 +819,22 @@ afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 static int
 afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     call_count      = 0;
+        int                     index           = 0;
+        int                     lockee_no       = 0;
+        int                     copies          = 0;
+        int                     i               = -1;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
+        copies   = priv->child_count;
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
+        call_count = afr_lockee_locked_nodes_count (int_lock);
 
-        call_count = afr_locked_nodes_count (int_lock->entry_locked_nodes,
-                                             priv->child_count);
         int_lock->lk_call_count = call_count;
 
         if (!call_count){
@@ -738,18 +844,23 @@ afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
                 goto out;
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->entry_locked_nodes[i] & LOCKED_YES) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
+        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+                lockee_no = i / copies;
+                index     = i % copies;
+                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                              AFR_UNLOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
                                            (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+                                           priv->children[index],
+                                           priv->children[index]->fops->entrylk,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
 
                         if (!--call_count)
                                 break;
@@ -763,15 +874,22 @@ out:
 
 static int32_t
 afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int child_index = (long) cookie;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     cky             = (long) cookie;
+        int                     child_index     = 0;
+        int                     lockee_no       = 0;
 
+        priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
+        child_index = ((int)cky) % priv->child_count;
+        lockee_no   = ((int)cky) / priv->child_count;
+
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
@@ -787,6 +905,8 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_errno              = op_errno;
                         int_lock->lock_op_errno      = op_errno;
                 }
+
+		int_lock->lk_attempted_count++;
         }
         UNLOCK (&frame->lock);
 
@@ -795,10 +915,17 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 afr_unlock (frame, this);
         } else {
                 if (op_ret == 0) {
-                        int_lock->locked_nodes[child_index] |= LOCKED_YES;
-                        int_lock->lock_count++;
+                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lockee[lockee_no].locked_count++;
+                                int_lock->entrylk_lock_count++;
+                        } else {
+                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lock_count++;
+                        }
                 }
-                afr_lock_blocking (frame, this, child_index + 1);
+                afr_lock_blocking (frame, this, cky + 1);
         }
 
         return 0;
@@ -806,98 +933,26 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 static int32_t
 afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_inodelk_out (frame, AFR_INODELK_TRANSACTION,
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long) cookie);
 
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
         return 0;
 
 }
 
 static int32_t
-afr_lock_lower_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_private_t       *priv        = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-        int child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-
-                                local->op_ret   = op_ret;
-                        }
-
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret != 0) {
-                afr_unlock (frame, this);
-                goto out;
-        } else {
-                int_lock->lower_locked_nodes[child_index] |= LOCKED_LOWER;
-                int_lock->lock_count++;
-        }
-
-        /* The lower path has been locked. Now lock the higher path */
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                              AFR_LOCK_OP, higher_name, child_index);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->entrylk,
-                           this->name, higher, higher_name,
-                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-out:
-        return 0;
-}
-
-static int32_t
 afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long)cookie);
 
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -905,6 +960,7 @@ static int
 afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
 
@@ -915,18 +971,16 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                memcpy (int_lock->inode_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->inodelk_lock_count = int_lock->lock_count;
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                memcpy (inodelk->locked_nodes, int_lock->locked_nodes,
+                        sizeof (*inodelk->locked_nodes) * priv->child_count);
+                inodelk->lock_count = int_lock->lock_count;
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
         case AFR_ENTRY_TRANSACTION:
-                memcpy (int_lock->entry_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->entrylk_lock_count = int_lock->lock_count;
+                /*entrylk_count is being used in both non-blocking and blocking
+                 * modes */
                 break;
         }
 
@@ -934,25 +988,67 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 
 }
 
+static inline gf_boolean_t
+afr_is_entrylk (afr_internal_lock_t *int_lock,
+                afr_transaction_type trans_type)
+{
+        gf_boolean_t is_entrylk = _gf_false;
+
+        if ((int_lock->transaction_lk_type == AFR_SELFHEAL_LK) &&
+            int_lock->selfheal_lk_type == AFR_ENTRY_SELF_HEAL_LK) {
+
+                is_entrylk = _gf_true;
+
+        } else if ((int_lock->transaction_lk_type == AFR_TRANSACTION_LK) &&
+                 (trans_type == AFR_ENTRY_TRANSACTION ||
+                  trans_type == AFR_ENTRY_RENAME_TRANSACTION)) {
+
+                is_entrylk = _gf_true;
+
+        } else {
+                is_entrylk = _gf_false;
+        }
+
+        return is_entrylk;
+}
+
+static gf_boolean_t
+_is_lock_wind_needed (afr_local_t *local, int child_index)
+{
+        if (!local->child_up[child_index])
+                return _gf_false;
+
+        return _gf_true;
+}
+
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
+afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
 {
         afr_internal_lock_t *int_lock    = NULL;
+        afr_inodelk_t       *inodelk     = NULL;
         afr_local_t         *local       = NULL;
         afr_private_t       *priv        = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
         struct gf_flock flock = {0,};
         uint64_t ctx = 0;
         int ret = 0;
+        int child_index = 0;
+        int lockee_no   = 0;
+        gf_boolean_t is_entrylk = _gf_false;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
+        local         = frame->local;
+        int_lock      = &local->internal_lock;
+        priv          = this->private;
+        child_index   = cookie % priv->child_count;
+        lockee_no     = cookie / priv->child_count;
+        is_entrylk    = afr_is_entrylk (int_lock, local->transaction.type);
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
+
+        if (!is_entrylk) {
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                flock.l_start = inodelk->flock.l_start;
+                flock.l_len   = inodelk->flock.l_len;
+                flock.l_type  = inodelk->flock.l_type;
+        }
 
         if (local->fd) {
                 ret = fd_ctx_get (local->fd, this, &ctx);
@@ -971,42 +1067,26 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
 
                         return 0;
                 }
-
-                /* skip over children that or down
-                   or don't have the fd open */
-
-                while ((child_index < priv->child_count)
-                       && (!local->child_up[child_index] ||
-                          !local->fd_open_on[child_index]))
-
-                        child_index++;
-        } else {
-                /* skip over children that are down */
-                while ((child_index < priv->child_count)
-                       && !local->child_up[child_index])
-                        child_index++;
         }
 
-        if ((child_index == priv->child_count) &&
-            int_lock->lock_count == 0) {
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "unable to lock on even one child");
-
-                local->op_ret           = -1;
-                int_lock->lock_op_ret   = -1;
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+                if ((is_entrylk && int_lock->entrylk_lock_count == 0) ||
+                    (!is_entrylk && int_lock->lock_count == 0)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to lock on even one child");
 
-                afr_copy_locked_nodes (frame, this);
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
 
-                afr_unlock(frame, this);
+                        afr_copy_locked_nodes (frame, this);
 
-                return 0;
+                        afr_unlock(frame, this);
 
+                        return 0;
+                }
         }
 
-        if ((child_index == priv->child_count)
-            || (int_lock->lock_count == int_lock->lk_expected_count)) {
-
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
                 /* we're done locking */
 
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -1019,12 +1099,18 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                 return 0;
         }
 
+        if (!_is_lock_wind_needed (local, child_index)) {
+                afr_lock_blocking (frame, this, cookie + 1);
+                return 0;
+        }
+
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
 
                 if (local->fd) {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
                                               AFR_LOCK_OP, &flock, F_SETLKW,
                                               child_index);
 
@@ -1032,11 +1118,12 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                            (void *) (long) child_index,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLKW, &flock);
+                                           int_lock->domain, local->fd,
+                                           F_SETLKW, &flock, NULL);
 
                 } else {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
                                               AFR_LOCK_OP, &flock, F_SETLKW,
                                               child_index);
 
@@ -1044,63 +1131,44 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                            (void *) (long) child_index,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLKW, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLKW, &flock, NULL);
                 }
 
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
-        {
-                lower = lower_path (&local->transaction.parent_loc,
-                                    local->transaction.basename,
-                                    &local->transaction.new_parent_loc,
-                                    local->transaction.new_basename);
-
-                lower_name = (lower == &local->transaction.parent_loc ?
-                              local->transaction.basename :
-                              local->transaction.new_basename);
-
-                afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                      AFR_LOCK_OP, lower_name, child_index);
-
-
-                STACK_WIND_COOKIE (frame, afr_lock_lower_cbk,
-                                   (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->entrylk,
-                                   this->name, lower, lower_name,
-                                   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-                break;
-        }
-
         case AFR_ENTRY_TRANSACTION:
+                /*Accounting for child_index increments on 'down'
+                 *and 'fd-less' children */
+
                 if (local->fd) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
+                                              AFR_LOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              cookie);
 
                         STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
+                                           (void *) (long) cookie,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->fentrylk,
-                                           this->name, local->fd,
-                                           local->transaction.basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
+                                           int_lock->domain, local->fd,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
                 } else {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
+                        AFR_TRACE_ENTRYLK_IN (frame, this,
+                                              AFR_ENTRYLK_TRANSACTION,
                                               AFR_LOCK_OP, local->transaction.basename,
                                               child_index);
 
                         STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
+                                           (void *) (long) cookie,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->entrylk,
-                                           this->name,
-                                           &local->transaction.parent_loc,
-                                           local->transaction.basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
                 }
 
                 break;
@@ -1128,11 +1196,11 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
-                up_count = afr_up_children_count (local->child_up,
-                                                  priv->child_count);
-                int_lock->lk_expected_count = 2 * up_count;
-                //fallthrough
         case AFR_ENTRY_TRANSACTION:
+                up_count = AFR_COUNT (local->child_up, priv->child_count);
+                int_lock->lk_call_count = int_lock->lk_expected_count
+                                        = (int_lock->lockee_count *
+                                           up_count);
                 initialize_entrylk_variables (frame, this);
                 break;
         }
@@ -1144,42 +1212,55 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
 
 static int32_t
 afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
         int call_count          = 0;
         int child_index         = (long) cookie;
+        int copies              = 0;
+        int index               = 0;
+        int lockee_no           = 0;
+        afr_private_t       *priv = NULL;
+
+        priv = this->private;
+
+        copies = priv->child_count;
+        index = child_index % copies;
+        lockee_no = child_index / copies;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_LOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
                                op_errno, (long) cookie);
 
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret < 0 ) {
-                if (op_errno == ENOSYS) {
+	LOCK (&frame->lock);
+	{
+		if (op_ret < 0 ) {
+			if (op_errno == ENOSYS) {
                         /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret         = op_ret;
-                        int_lock->lock_op_ret = op_ret;
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on server");
+				local->op_ret         = op_ret;
+				int_lock->lock_op_ret = op_ret;
+
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+		} else if (op_ret == 0) {
+			int_lock->lockee[lockee_no].locked_nodes[index] |= \
+				LOCKED_YES;
+			int_lock->lockee[lockee_no].locked_count++;
+			int_lock->entrylk_lock_count++;
+		}
 
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-        } else if (op_ret == 0) {
-                int_lock->entry_locked_nodes[child_index] |= LOCKED_YES;
-                int_lock->entrylk_lock_count++;
+                call_count = --int_lock->lk_call_count;
         }
+        UNLOCK (&frame->lock);
 
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
@@ -1206,42 +1287,26 @@ afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-void
-afr_mark_fd_open_on (afr_local_t *local, afr_fd_ctx_t *fd_ctx,
-                       size_t child_count)
-{
-        int             i = 0;
-
-        GF_ASSERT (local->fd_open_on);
-
-        memset (local->fd_open_on, 0, sizeof (*local->fd_open_on)*child_count);
-        for (i = 0; i < child_count; i++)
-                if (fd_ctx->opened_on[i] == AFR_FD_OPENED)
-                        local->fd_open_on[i] = 1;
-}
-
 int
 afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int32_t call_count = 0;
+        afr_internal_lock_t *int_lock   = NULL;
+        afr_local_t         *local      = NULL;
+        afr_private_t       *priv       = NULL;
+        afr_fd_ctx_t        *fd_ctx     = NULL;
+        int                 copies      = 0;
+        int                 index       = 0;
+        int                 lockee_no   = 0;
+        int32_t             call_count  = 0;
         int i = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
+        copies = priv->child_count;
         initialize_entrylk_variables (frame, this);
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
         if (local->fd) {
                 fd_ctx = afr_fd_ctx_get (local->fd, this);
                 if (!fd_ctx) {
@@ -1254,11 +1319,11 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         return -1;
                 }
 
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
-                call_count = internal_lock_count (frame, this);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
 
@@ -1271,42 +1336,52 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 
                 /* Send non-blocking entrylk calls only on up children
                    and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i]) {
-                                afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fentrylk,
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->fentrylk,
                                                    this->name, local->fd,
-                                                   basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
+                                if (!--call_count)
+                                        break;
                         }
                 }
         } else {
-                GF_ASSERT (loc);
-
-                call_count = internal_lock_count (frame, this);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
 
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->entrylk,
-                                                   this->name, loc, basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->entrylk,
+                                                   this->name, &int_lock->lockee[lockee_no].loc,
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
 
                                 if (!--call_count)
                                         break;
-
                         }
                 }
         }
@@ -1316,70 +1391,69 @@ out:
 
 int32_t
 afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         int call_count  = 0;
         int child_index = (long) cookie;
         afr_fd_ctx_t        *fd_ctx = NULL;
-        afr_private_t       *priv     = NULL;
 
 
-        priv = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        afr_trace_inodelk_out (frame, AFR_INODELK_NB_TRANSACTION,
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long) cookie);
 
+	if (local->fd)
+		fd_ctx = afr_fd_ctx_get (local->fd, this);
+
         LOCK (&frame->lock);
         {
+		if (op_ret < 0) {
+			if (op_errno == ENOSYS) {
+				/* return ENOTSUP */
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on "
+					"server");
+				local->op_ret                = op_ret;
+				int_lock->lock_op_ret        = op_ret;
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+			if (local->transaction.eager_lock)
+				local->transaction.eager_lock[child_index] = 0;
+		} else {
+			inodelk->locked_nodes[child_index] |= LOCKED_YES;
+			inodelk->lock_count++;
+
+			if (local->transaction.eager_lock &&
+			    local->transaction.eager_lock[child_index] &&
+			    local->fd) {
+				/* piggybacked */
+				if (op_ret == 1) {
+					/* piggybacked */
+				} else if (op_ret == 0) {
+					/* lock acquired from server */
+                                        fd_ctx->lock_acquired[child_index]++;
+				}
+			}
+		}
+
                 call_count = --int_lock->lk_call_count;
         }
         UNLOCK (&frame->lock);
 
-        if (op_ret < 0) {
-                if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret                = op_ret;
-                        int_lock->lock_op_ret        = op_ret;
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-        } else {
-                int_lock->inode_locked_nodes[child_index]
-                        |= LOCKED_YES;
-                int_lock->inodelk_lock_count++;
-
-                if (priv->eager_lock && local->fd) {
-                        fd_ctx = afr_fd_ctx_get (local->fd, this);
-                        local->transaction.eager_lock[child_index] = 1;
-                        /* piggybacked */
-
-                        if (op_ret == 1) {
-                                /* piggybacked */
-                        } else if (op_ret == 0) {
-                                /* lock acquired from server */
-                                LOCK (&local->fd->lock);
-                                {
-                                        fd_ctx->lock_acquired[child_index]++;
-                                }
-                                UNLOCK (&local->fd->lock);
-                        }
-                }
-        }
-
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "Last inode locking reply received");
                 /* all locks successful. Proceed to call FOP */
-                if (int_lock->inodelk_lock_count ==
-                                int_lock->lk_expected_count) {
+                if (inodelk->lock_count == int_lock->lk_expected_count) {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "All servers locked. Calling the cbk");
                         int_lock->lock_op_ret = 0;
@@ -1403,30 +1477,29 @@ int
 afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         afr_fd_ctx_t        *fd_ctx   = NULL;
-        int32_t  call_count = 0;
-        int      i          = 0;
-        int      ret        = 0;
-        struct gf_flock flock = {0,};
-        struct gf_flock full_flock = {0,};
-        struct gf_flock *flock_use = &flock;
-        int     piggyback = 0;
+        int32_t             call_count = 0;
+        int                 i          = 0;
+        int                 ret        = 0;
+        struct              gf_flock flock = {0,};
+        struct              gf_flock full_flock = {0,};
+        struct              gf_flock *flock_use = NULL;
+        int                 piggyback = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        gf_log (this->name, GF_LOG_DEBUG, "attempting data lock range %"PRIu64
-                " %"PRIu64" by %s", flock.l_start, flock.l_len,
-                lkowner_utoa (&frame->root->lk_owner));
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = inodelk->flock.l_type;
 
-        full_flock.l_type = int_lock->lk_flock.l_type;
+        full_flock.l_type = inodelk->flock.l_type;
 
         initialize_inodelk_variables (frame, this);
 
@@ -1442,11 +1515,11 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         ret = -1;
                         goto out;
                 }
 
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
                 call_count = internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
@@ -1461,14 +1534,18 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                 /* Send non-blocking inodelk calls only on up children
                    and where the fd has been opened */
                 for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i] || !local->fd_open_on[i])
+                        if (!local->child_up[i])
                                 continue;
 
-                        if (!priv->eager_lock)
+                        flock_use = &flock;
+                        if (!local->transaction.eager_lock_on) {
                                 goto wind;
+                        }
 
-                        flock_use = &full_flock;
                         piggyback = 0;
+                        local->transaction.eager_lock[i] = 1;
+
+			afr_set_delayed_post_op (frame, this);
 
                         LOCK (&local->fd->lock);
                         {
@@ -1482,21 +1559,23 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         if (piggyback) {
                                 /* (op_ret == 1) => indicate piggybacked lock */
                                 afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
-                                                             this, 1, 0);
+                                                             this, 1, 0, NULL);
                                 if (!--call_count)
                                         break;
                                 continue;
                         }
+                        flock_use = &full_flock;
                 wind:
-                        afr_trace_inodelk_in (frame, AFR_INODELK_NB_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_NB_TRANSACTION,
                                               AFR_LOCK_OP, flock_use, F_SETLK, i);
 
                         STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLK, flock_use);
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
 
                         if (!--call_count)
                                 break;
@@ -1509,15 +1588,16 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                 for (i = 0; i < priv->child_count; i++) {
                         if (!local->child_up[i])
                                 continue;
-                        afr_trace_inodelk_in (frame, AFR_INODELK_NB_TRANSACTION,
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_NB_TRANSACTION,
                                               AFR_LOCK_OP, &flock, F_SETLK, i);
 
                         STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLK, &flock);
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -1527,200 +1607,6 @@ out:
         return ret;
 }
 
-static int
-__is_lower_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i     = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-__is_higher_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->locked_nodes[i] & LOCKED_YES)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-afr_unlock_lower_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-        priv  = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = __is_lower_locked (frame, this);
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-
-                        if (!--call_count)
-                                break;
-
-                }
-        }
-
-out:
-        return 0;
-
-}
-
-
-static int
-afr_post_unlock_higher_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.done (frame, this);
-        return 0;
-}
-
-static int
-afr_post_unlock_lower_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        if (__is_higher_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking higher");
-                int_lock->lk_basename = higher_name;
-                int_lock->lk_loc      = higher;
-                int_lock->lock_cbk    = afr_post_unlock_higher_cbk;
-
-                afr_unlock_entrylk (frame, this);
-        } else
-                local->transaction.done (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        if (__is_lower_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking lower");
-                int_lock->lk_basename = lower_name;
-                int_lock->lk_loc      = lower;
-                int_lock->lock_cbk    = afr_post_unlock_lower_cbk;
-
-                afr_unlock_lower_entrylk (frame, this);
-        } else
-                afr_post_unlock_lower_cbk (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_transaction (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        return (local->transaction.type ==
-                AFR_ENTRY_RENAME_TRANSACTION);
-
-}
-
 int32_t
 afr_unlock (call_frame_t *frame, xlator_t *this)
 {
@@ -1732,10 +1618,8 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
                 if (is_afr_lock_transaction (local))
                         afr_unlock_inodelk (frame, this);
                 else
-                        if (!afr_rename_transaction (frame, this))
-                                afr_unlock_entrylk (frame, this);
-                        else
-                                afr_rename_unlock (frame, this);
+                        afr_unlock_entrylk (frame, this);
+
         } else {
                 if (is_afr_lock_selfheal (local))
                         afr_unlock_inodelk (frame, this);
@@ -1747,513 +1631,37 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
 }
 
 int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes)
-{
-        afr_private_t *priv  = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        uint64_t       tmp   = 0;
-        int            ret   = 0;
-
-        priv = this->private;
-
-        ret = afr_fd_ctx_set (this, fd);
-        if (ret)
-                goto out;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "failed to get the fd ctx");
-                goto out;
-        }
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        GF_ASSERT (fdctx->locked_on);
-
-        memcpy (fdctx->locked_on, locked_nodes,
-                priv->child_count);
-
-out:
-        return ret;
-}
-
-static int
-__is_fd_saved (xlator_t *this, fd_t *fd)
-{
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_private_t   *priv      = NULL;
-        int              found     = 0;
-
-        priv = this->private;
-
-        list_for_each_entry (locked_fd, &priv->saved_fds, list) {
-                if (locked_fd->fd == fd) {
-                        found = 1;
-                        break;
-                }
-        }
-
-        return found;
-}
-
-static int
-__afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-        int              ret       = 0;
-
-        priv = this->private;
-
-        locked_fd = GF_CALLOC (1, sizeof (*locked_fd),
-                               gf_afr_mt_locked_fd);
-        if (!locked_fd) {
-                ret = -1;
-                goto out;
-        }
-
-        locked_fd->fd = fd;
-        INIT_LIST_HEAD (&locked_fd->list);
-
-        list_add_tail (&locked_fd->list, &priv->saved_fds);
-
-out:
-        return ret;
-}
-
-int
-afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        int              ret       = 0;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                if (__is_fd_saved (this, fd)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p already saved", fd);
-                        goto unlock;
-                }
-
-                ret = __afr_save_locked_fd (this, fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "fd=%p could not be saved", fd);
-                        goto unlock;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->mutex);
-
-        return ret;
-}
-
-static int
-afr_lock_recovery_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-
-        local = frame->local;
-
-        locked_fd = local->locked_fd;
-
-        STACK_DESTROY (frame->root);
-        afr_local_cleanup (local, this);
-
-        afr_save_locked_fd (this, locked_fd->fd);
-
-        return 0;
-
-}
-
-static int
-afr_get_source_lock_recovery (xlator_t *this, fd_t *fd)
-{
-        afr_fd_ctx_t  *fdctx        = NULL;
-        afr_private_t *priv         = NULL;
-        uint64_t      tmp           = 0;
-        int           i             = 0;
-        int           source_child  = -1;
-        int           ret           = 0;
-
-        priv = this->private;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (fdctx->locked_on[i]) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Found lock recovery source=%d", i);
-                        source_child = i;
-                        break;
-                }
-        }
-
-out:
-        return source_child;
-
-}
-
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock);
-int32_t
-afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t source_child = 0;
-        struct gf_flock flock   = {0,};
-
-        local = frame->local;
-        priv  = this->private;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "lock recovery failed");
-                goto cleanup;
-        }
-
-        source_child = local->source_child;
-
-        memcpy (&flock, lock, sizeof (*lock));
-
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-int
-afr_recover_lock (call_frame_t *frame, xlator_t *this,
-                  struct gf_flock *flock)
-{
-        afr_local_t   *local             = NULL;
-        afr_private_t *priv              = NULL;
-        int32_t      lock_recovery_child = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        lock_recovery_child = local->lock_recovery_child;
-
-        frame->root->lk_owner = flock->l_owner;
-
-        STACK_WIND_COOKIE (frame, afr_recover_lock_cbk,
-                           (void *) (long) lock_recovery_child,
-                           priv->children[lock_recovery_child],
-                           priv->children[lock_recovery_child]->fops->lk,
-                           local->fd, F_SETLK, flock);
-
-        return 0;
-}
-
-static int
-is_afr_lock_eol (struct gf_flock *lock)
-{
-        int ret = 0;
-
-        if ((lock->l_type == GF_LK_EOL))
-                ret = 1;
-
-        return ret;
-}
-
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Failed to get locks on fd");
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Got a lock on fd");
-
-        if (is_afr_lock_eol (lock)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Reached EOL on locks on fd");
-                goto cleanup;
-        }
-
-        afr_recover_lock (frame, this, lock);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-
-        return 0;
-}
-
-static int
-afr_lock_recovery (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local        = NULL;
-        afr_private_t *priv         = NULL;
-        fd_t          *fd           = NULL;
-        int            ret          = 0;
-        int32_t        source_child = 0;
-        struct gf_flock   flock        = {0,};
-
-        priv  = this->private;
-        local = frame->local;
-
-        fd = local->fd;
-
-        source_child = afr_get_source_lock_recovery (this, fd);
-        if (source_child < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not recover locks due to lock "
-                        "split brain");
-                ret = -1;
-                goto out;
-        }
-
-        local->source_child = source_child;
-
-        /* the flock can be zero filled as we're querying incrementally
-           the locks held on the fd.
-        */
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
-
-out:
-        return ret;
-}
-
-
-static int
-afr_mark_fd_opened (xlator_t *this, fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp   = 0;
-        int           ret   = 0;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        fdctx->opened_on[child_index] = AFR_FD_OPENED;
-
-out:
-        return ret;
-}
-
-int32_t
-afr_lock_recovery_preopen_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        int32_t child_index = (long )cookie;
-        int ret = 0;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Reopen during lock-recovery failed");
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Open succeeded => proceed to recover locks");
-
-        ret = afr_lock_recovery (frame, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Lock recovery failed");
-                goto cleanup;
-        }
-
-        ret = afr_mark_fd_opened (this, fd, child_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Marking fd open failed");
-                goto cleanup;
-        }
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-static int
-afr_lock_recovery_preopen (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
-        uint64_t       tmp   = 0;
-        afr_fd_ctx_t  *fdctx = NULL;
-        loc_t          loc   = {0,};
-        int32_t        child_index = 0;
-        int            ret   = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        GF_ASSERT (local && local->fd);
-
-        ret = fd_ctx_get (local->fd, this, &tmp);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get the context of fd",
-                        uuid_utoa (local->fd->inode->gfid));
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-        /* TODO: instead we should return from the function */
-        GF_ASSERT (fdctx);
-
-        child_index = local->lock_recovery_child;
-
-        inode_path (local->fd->inode, NULL, (char **)&loc.path);
-        loc.name   = strrchr (loc.path, '/');
-        loc.inode  = inode_ref (local->fd->inode);
-        loc.parent = inode_parent (local->fd->inode, 0, NULL);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_recovery_preopen_cbk,
-                           (void *)(long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->open,
-                           &loc, fdctx->flags, local->fd,
-                           fdctx->wbflags);
-
-        return 0;
-}
-
-static int
-is_fd_opened (fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp = 0;
-        int           ret = 0;
-
-        ret = fd_ctx_get (fd, THIS, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        if (fdctx->opened_on[child_index] == AFR_FD_OPENED)
-                ret = 1;
-
-out:
-        return ret;
-}
-
-int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index)
-{
-        call_frame_t    *frame      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_local_t     *local      = NULL;
-        afr_locked_fd_t *locked_fd  = NULL;
-        afr_locked_fd_t  *tmp       = NULL;
-        int              ret        = -1;
-        struct list_head locks_list = {0,};
-        int32_t          op_errno   = 0;
-
-
-        priv = this->private;
-
-        if (list_empty (&priv->saved_fds))
-                goto out;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0) {
-                ret = -1;
-                goto out;
-        }
-
-        frame->local = local;
-
-        INIT_LIST_HEAD (&locks_list);
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                list_splice_init (&priv->saved_fds, &locks_list);
-        }
-        pthread_mutex_unlock (&priv->mutex);
-
-        list_for_each_entry_safe (locked_fd, tmp,
-                                  &locks_list, list) {
-
-                list_del_init (&locked_fd->list);
-
-                local->fd                  = fd_ref (locked_fd->fd);
-                local->lock_recovery_child = child_index;
-                local->locked_fd           = locked_fd;
-
-                if (!is_fd_opened (locked_fd->fd, child_index)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting open before lock "
-                                "recovery");
-                        afr_lock_recovery_preopen (frame, this);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting lock recovery "
-                                "without a preopen");
-                        afr_lock_recovery (frame, this);
-                }
-        }
-
-out:
-        if ((ret < 0) && frame)
-                AFR_STACK_DESTROY (frame);
-        return ret;
-}
-
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
                           unsigned int child_count)
 {
-        afr_local_t *dst_local = NULL;
-        afr_local_t *src_local = NULL;
-        afr_internal_lock_t *dst_lock = NULL;
-        afr_internal_lock_t *src_lock = NULL;
+        afr_local_t         *dst_local   = NULL;
+        afr_local_t         *src_local   = NULL;
+        afr_internal_lock_t *dst_lock    = NULL;
+        afr_internal_lock_t *src_lock    = NULL;
+        afr_inodelk_t       *dst_inodelk = NULL;
+        afr_inodelk_t       *src_inodelk = NULL;
+        int                 ret = -1;
 
-        dst_local = dst->local;
-        dst_lock  = &dst_local->internal_lock;
         src_local = src->local;
         src_lock  = &src_local->internal_lock;
-        if (src_lock->inode_locked_nodes) {
-                memcpy (dst_lock->inode_locked_nodes,
-                        src_lock->inode_locked_nodes,
-                        sizeof (*dst_lock->inode_locked_nodes) * child_count);
-                memset (src_lock->inode_locked_nodes, 0,
-                        sizeof (*src_lock->inode_locked_nodes) * child_count);
-        }
-
-        dst_lock->inodelk_lock_count = src_lock->inodelk_lock_count;
-        src_lock->inodelk_lock_count = 0;
+        src_inodelk = afr_get_inodelk (src_lock, dom);
+        dst_local = dst->local;
+        dst_lock  = &dst_local->internal_lock;
+        dst_inodelk = afr_get_inodelk (dst_lock, dom);
+        if (!dst_inodelk || !src_inodelk)
+                goto out;
+        if (src_inodelk->locked_nodes) {
+                memcpy (dst_inodelk->locked_nodes, src_inodelk->locked_nodes,
+                        sizeof (*dst_inodelk->locked_nodes) * child_count);
+                memset (src_inodelk->locked_nodes, 0,
+                        sizeof (*src_inodelk->locked_nodes) * child_count);
+        }
+
+        dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
+        dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
+        dst_inodelk->lock_count = src_inodelk->lock_count;
+        src_inodelk->lock_count = 0;
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index 343260a79..05df90cc0 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -49,6 +40,9 @@ enum gf_afr_mem_types_ {
         gf_afr_mt_shd_timer_t,
         gf_afr_mt_shd_event_t,
         gf_afr_mt_time_t,
+        gf_afr_mt_pos_data_t,
+	gf_afr_mt_reply_t,
+	gf_afr_mt_subvol_healer_t,
         gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index a203a36f9..f86aa7fd8 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -52,87 +43,31 @@
 #include "afr-dir-read.h"
 #include "afr-dir-write.h"
 #include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
 
-int
-afr_stale_child_up (afr_local_t *local, xlator_t *this)
-{
-        int             i = 0;
-        afr_private_t   *priv = NULL;
-        int             up = -1;
 
-        priv = this->private;
-
-        if (!local->fresh_children)
-                local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children)
-                goto out;
-
-        afr_inode_get_read_ctx (this, local->fd->inode, local->fresh_children);
-        if (priv->child_count == afr_get_children_count (local->fresh_children,
-                                                         priv->child_count))
-                goto out;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-                if (afr_is_child_present (local->fresh_children,
-                                          priv->child_count, i))
-                        continue;
-                up = i;
-                break;
-        }
-out:
-        return up;
-}
-
-void
-afr_perform_data_self_heal (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        inode_t         *inode = NULL;
-        int             st_child = -1;
-        char            reason[64] = {0};
-
-        local = frame->local;
-        sh = &local->self_heal;
-        inode = local->fd->inode;
-
-        if (!IA_ISREG (inode->ia_type))
-                goto out;
-
-        st_child = afr_stale_child_up (local, this);
-        if (st_child < 0)
-                goto out;
-
-        sh->do_data_self_heal          = _gf_true;
-        sh->do_metadata_self_heal      = _gf_true;
-        sh->do_gfid_self_heal          = _gf_true;
-        sh->do_missing_entry_self_heal = _gf_true;
-
-        snprintf (reason, sizeof (reason), "stale subvolume %d detected",
-                  st_child);
-        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
-                              reason, NULL, NULL);
-out:
-        return;
+        if (!fd || !fd->inode)
+                return _gf_false;
+        else if (fd_is_anonymous (fd))
+                return _gf_false;
+        else if (uuid_is_null (fd->inode->gfid))
+                return _gf_false;
+
+        return _gf_true;
 }
 
+
 int
 afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+                        struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t * local = frame->local;
-        afr_private_t *priv = NULL;
 
-        priv = this->private;
-        if (afr_open_only_data_self_heal (priv->data_self_heal))
-                afr_perform_data_self_heal (frame, this);
         AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                          local->fd);
+                          local->fd, xdata);
         return 0;
 }
 
@@ -140,53 +75,41 @@ afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_open_cbk (call_frame_t *frame, void *cookie,
               xlator_t *this, int32_t op_ret, int32_t op_errno,
-              fd_t *fd)
+              fd_t *fd, dict_t *xdata)
 {
         afr_local_t *  local       = NULL;
-        int            ret         = 0;
         int            call_count  = -1;
         int            child_index = (long) cookie;
-        afr_private_t *priv        = NULL;
+	afr_fd_ctx_t  *fd_ctx = NULL;
 
-        priv = this->private;
         local = frame->local;
+	fd_ctx = local->fd_ctx;
 
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
                         local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0) {
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+                } else {
                         local->op_ret = op_ret;
-                        local->success_count++;
-
-                        ret = afr_child_fd_ctx_set (this, fd, child_index,
-                                                    local->cont.open.flags,
-                                                    local->cont.open.wbflags);
-                        if (ret) {
-                                local->op_ret = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                 }
         }
-unlock:
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                if ((local->cont.open.flags & O_TRUNC)
-                    && (local->op_ret >= 0)) {
+                if ((fd_ctx->flags & O_TRUNC) && (local->op_ret >= 0)) {
                         STACK_WIND (frame, afr_open_ftruncate_cbk,
                                     this, this->fops->ftruncate,
-                                    fd, 0);
+                                    fd, 0, NULL);
                 } else {
-                        if (afr_open_only_data_self_heal (priv->data_self_heal))
-                                afr_perform_data_self_heal (frame, this);
                         AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd);
+                                          local->op_errno, local->fd,
+					  local->xdata_rsp);
                 }
         }
 
@@ -195,21 +118,16 @@ unlock:
 
 int
 afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+          fd_t *fd, dict_t *xdata)
 {
         afr_private_t * priv       = NULL;
         afr_local_t *   local      = NULL;
         int             i          = 0;
-        int             ret        = -1;
         int32_t         call_count = 0;
         int32_t         op_errno   = 0;
-        int32_t         wind_flags = flags & (~O_TRUNC);
-        //We can't let truncation to happen outside transaction.
+	afr_fd_ctx_t   *fd_ctx = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+        //We can't let truncation to happen outside transaction.
 
         priv = this->private;
 
@@ -217,239 +135,203 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                 QUORUM_CHECK(open,out);
         }
 
-        if (afr_is_split_brain (this, loc->inode)) {
-                /* self-heal failed */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to open as split brain seen, returning EIO");
-                op_errno = EIO;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
-        call_count   = local->call_count;
-        loc_copy (&local->loc, loc);
+        local->fd = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
+	fd_ctx->flags = flags;
 
-        local->cont.open.flags   = flags;
-        local->cont.open.wbflags = wbflags;
+        call_count = local->call_count;
 
-        local->fd = fd_ref (fd);
+        local->cont.open.flags = flags;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           loc, wind_flags, fd, wbflags);
-
+                                           loc, (flags & ~O_TRUNC), fd, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (open, frame, -1, op_errno, fd);
+	AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, NULL);
 
         return 0;
 }
 
-//NOTE: this function should be called with holding the lock on
-//fd to which fd_ctx belongs
-void
-afr_get_resumable_calls (xlator_t *this, afr_fd_ctx_t *fd_ctx,
-                         struct list_head *list)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-        afr_private_t         *priv        = NULL;
-        int                    i = 0;
-        gf_boolean_t           call = _gf_false;
-
-        priv = this->private;
-        list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                  call_list) {
-                call = _gf_true;
-                call_local = paused_call->frame->local;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (call_local->child_up[i] &&
-                            (fd_ctx->opened_on[i] == AFR_FD_OPENING))
-                                call = _gf_false;
-                }
-
-                if (call) {
-                        list_del_init (&paused_call->call_list);
-                        list_add (&paused_call->call_list, list);
-                }
-        }
-}
-
-void
-afr_resume_calls (xlator_t *this, struct list_head *list)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-
-        list_for_each_entry_safe (paused_call, tmp, list, call_list) {
-                list_del_init (&paused_call->call_list);
-                call_local = paused_call->frame->local;
-                call_local->fop_call_continue (paused_call->frame, this);
-                GF_FREE (paused_call);
-        }
-}
-
 int
 afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, fd_t *fd)
+                         int32_t op_ret, int32_t op_errno, fd_t *fd,
+                         dict_t *xdata)
 {
-        afr_local_t           *local       = NULL;
-        afr_private_t         *priv        = NULL;
-        afr_fd_ctx_t          *fd_ctx      = NULL;
-        int                    call_count  = 0;
-        int                    child_index = (long) cookie;
-        struct list_head       paused_calls = {0};
-        gf_boolean_t           fop_paused = _gf_false;
-        fd_t                   *local_fd  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        afr_fd_ctx_t  *fd_ctx     = NULL;
+        int           call_count  = 0;
+        int           child_index = (long) cookie;
 
         priv     = this->private;
         local    = frame->local;
-        fop_paused = local->fop_paused;
 
         if (op_ret >= 0) {
-                gf_log (this->name, GF_LOG_INFO, "fd for %s opened "
+                gf_log (this->name, GF_LOG_DEBUG, "fd for %s opened "
                         "successfully on subvolume %s", local->loc.path,
                         priv->children[child_index]->name);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open %s "
+                        "on subvolume %s", local->loc.path,
+                        priv->children[child_index]->name);
         }
 
-        local_fd = fd_ref (local->fd);
-        call_count = afr_frame_return (frame);
-        //Note: Do not access any thing using the frame outside call_count 0
-
-        //Note: No frame locking needed for this block of code
-        fd_ctx = afr_fd_ctx_get (local_fd, this);
-        if (!fd_ctx) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get fd context, %p", local_fd);
-                goto out;
-        }
+        fd_ctx = local->fd_ctx;
 
-        LOCK (&local_fd->lock);
+        LOCK (&local->fd->lock);
         {
                 if (op_ret >= 0) {
                         fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
                 } else {
                         fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
                 }
-                if (call_count == 0) {
-                        INIT_LIST_HEAD (&paused_calls);
-                        afr_get_resumable_calls (this, fd_ctx, &paused_calls);
-                }
-        }
-        UNLOCK (&local_fd->lock);
-out:
-        if (call_count == 0) {
-                afr_resume_calls (this, &paused_calls);
-                //If the fop is paused then resume_calls will continue the fop
-                if (fop_paused)
-                        goto done;
-
-                if (local->fop_call_continue)
-                        local->fop_call_continue (frame, this);
-                else
-                        AFR_STACK_DESTROY (frame);
         }
+        UNLOCK (&local->fd->lock);
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
 
-done:
-        fd_unref (local_fd);
         return 0;
 }
 
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open)
+
+static int
+afr_fd_ctx_need_open (fd_t *fd, xlator_t *this, unsigned char *need_open)
 {
-        afr_local_t       *local = NULL;
-        afr_private_t     *priv  = NULL;
-        int               i      = 0;
-        call_frame_t      *open_frame = NULL;
-        afr_local_t      *open_local = NULL;
-        int               ret    = -1;
-        ia_type_t         ia_type = IA_INVAL;
-        int32_t           op_errno = 0;
-
-        GF_ASSERT (fd_ctx);
-        GF_ASSERT (need_open_count > 0);
-        GF_ASSERT (need_open);
+	afr_fd_ctx_t *fd_ctx = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int count = 0;
+
+	priv = this->private;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return 0;
+
+	LOCK (&fd->lock);
+	{
+		for (i = 0; i < priv->child_count; i++) {
+			if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
+			    priv->child_up[i]) {
+				fd_ctx->opened_on[i] = AFR_FD_OPENING;
+				need_open[i] = 1;
+				count++;
+			} else {
+				need_open[i] = 0;
+			}
+		}
+	}
+	UNLOCK (&fd->lock);
+
+	return count;
+}
+
+
+void
+afr_fix_open (fd_t *fd, xlator_t *this)
+{
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
+        call_frame_t  *frame   = NULL;
+        afr_local_t   *local   = NULL;
+        int           ret      = -1;
+        int32_t       op_errno = 0;
+        afr_fd_ctx_t  *fd_ctx  = NULL;
+	unsigned char *need_open = NULL;
+	int call_count = 0;
 
-        local = frame->local;
         priv  = this->private;
-        if (!local->fop_call_continue) {
-                open_frame = copy_frame (frame);
-                if (!open_frame) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-                AFR_LOCAL_ALLOC_OR_GOTO (open_frame->local, out);
-                open_local = open_frame->local;
-                ret = afr_local_init (open_local, priv, &op_errno);
-                if (ret < 0)
-                        goto out;
-                loc_copy (&open_local->loc, &local->loc);
-                open_local->fd = fd_ref (local->fd);
-        } else {
-                ret = 0;
-                open_frame = frame;
-                open_local = local;
-        }
 
-        open_local->call_count = need_open_count;
+        if (!afr_is_fd_fixable (fd))
+                goto out;
+
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
+                goto out;
+
+	need_open = alloca0 (priv->child_count);
+
+	call_count = afr_fd_ctx_need_open (fd, this, need_open);
+	if (!call_count)
+		goto out;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
+
+        local->fd = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
+
+        local->call_count = call_count;
 
         gf_log (this->name, GF_LOG_DEBUG, "need open count: %d",
-                need_open_count);
+                call_count);
 
-        ia_type = open_local->fd->inode->ia_type;
-        GF_ASSERT (ia_type != IA_INVAL);
         for (i = 0; i < priv->child_count; i++) {
                 if (!need_open[i])
                         continue;
-                if (IA_IFDIR == ia_type) {
+
+                if (IA_IFDIR == fd->inode->ia_type) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "opening fd for dir %s on subvolume %s",
                                 local->loc.path, priv->children[i]->name);
 
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
                                            (void*) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->opendir,
-                                           &open_local->loc, open_local->fd);
+                                           &local->loc, local->fd,
+                                           NULL);
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "opening fd for file %s on subvolume %s",
                                 local->loc.path, priv->children[i]->name);
 
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
                                            (void *)(long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           &open_local->loc, fd_ctx->flags,
-                                           open_local->fd, fd_ctx->wbflags);
+                                           &local->loc,
+                                           fd_ctx->flags & (~O_TRUNC),
+                                           local->fd, NULL);
                 }
 
+		if (!--call_count)
+			break;
         }
-        op_errno = 0;
-        ret = 0;
+
+	return;
 out:
-        if (op_errno)
-                ret = -op_errno;
-        if (ret && open_frame)
-                AFR_STACK_DESTROY (open_frame);
-        return ret;
+        if (frame)
+                AFR_STACK_DESTROY (frame);
 }
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
new file mode 100644
index 000000000..186f68c33
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -0,0 +1,239 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "afr.h"
+#include "afr-transaction.h"
+
+int
+afr_read_txn_next_subvol (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int subvol = -1;
+
+	local = frame->local;
+	priv = this->private;
+
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->readable[i]) {
+			/* don't even bother trying here.
+			   just mark as attempted and move on. */
+			local->read_attempted[i] = 1;
+			continue;
+		}
+
+		if (!local->read_attempted[i]) {
+			subvol = i;
+			break;
+		}
+	}
+
+	/* If no more subvols were available for reading, we leave
+	   @subvol as -1, which is an indication we have run out of
+	   readable subvols. */
+	if (subvol != -1)
+		local->read_attempted[subvol] = 1;
+	local->readfn (frame, this, subvol);
+
+	return 0;
+}
+
+
+int
+afr_read_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
+{
+	afr_local_t *local = NULL;
+	int read_subvol = 0;
+	int event_generation = 0;
+	inode_t *inode = NULL;
+	int ret = -1;
+
+	local = frame->local;
+	inode = local->inode;
+
+	if (err) {
+		local->op_errno = -err;
+		local->op_ret = -1;
+		read_subvol = -1;
+		goto readfn;
+	}
+
+	ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
+					      &event_generation,
+					      local->transaction.type);
+
+	if (ret == -1 || !event_generation) {
+		/* Even after refresh, we don't have a good
+		   read subvolume. Time to bail */
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		read_subvol = -1;
+		goto readfn;
+	}
+
+	read_subvol = afr_read_subvol_select_by_policy (inode, this,
+							local->readable);
+
+	if (read_subvol == -1) {
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto readfn;
+	}
+
+	if (local->read_attempted[read_subvol]) {
+		afr_read_txn_next_subvol (frame, this);
+		return 0;
+	}
+
+	local->read_attempted[read_subvol] = 1;
+readfn:
+	local->readfn (frame, this, read_subvol);
+
+	return 0;
+}
+
+
+int
+afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+
+	local = frame->local;
+
+	if (!local->refreshed) {
+		local->refreshed = _gf_true;
+		afr_inode_refresh (frame, this, local->inode,
+				   afr_read_txn_refresh_done);
+	} else {
+		afr_read_txn_next_subvol (frame, this);
+	}
+
+	return 0;
+}
+
+
+/* afr_read_txn_wipe:
+
+   clean internal variables in @local in order to make
+   it possible to call afr_read_txn() multiple times from
+   the same frame
+*/
+
+void
+afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	local->readfn = NULL;
+
+	if (local->inode)
+		inode_unref (local->inode);
+
+	for (i = 0; i < priv->child_count; i++) {
+		local->read_attempted[i] = 0;
+		local->readable[i] = 0;
+	}
+}
+
+
+/*
+  afr_read_txn:
+
+  This is the read transaction function. The way it works:
+
+  - Determine read-subvolume from inode ctx.
+
+  - If read-subvolume's generation was stale, refresh ctx once by
+    calling afr_inode_refresh()
+
+    Else make an attempt to read on read-subvolume.
+
+  - If attempted read on read-subvolume fails, refresh ctx once
+    by calling afr_inode_refresh()
+
+  - After ctx refresh, query read-subvolume freshly and attempt
+    read once.
+
+  - If read fails, try every other readable[] subvolume before
+    finally giving up. readable[] elements are set by afr_inode_refresh()
+    based on dirty and pending flags.
+
+  - If file is in split brain in the backend, generation will be
+    kept 0 by afr_inode_refresh() and readable[] will be set 0 for
+    all elements. Therefore reads always fail.
+*/
+
+int
+afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
+	      afr_read_txn_wind_t readfn, afr_transaction_type type)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int read_subvol = -1;
+	int event_generation = 0;
+	int ret = -1;
+
+	priv = this->private;
+	local = frame->local;
+
+	afr_read_txn_wipe (frame, this);
+
+	local->readfn = readfn;
+	local->inode = inode_ref (inode);
+
+	local->transaction.type = type;
+	ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
+					      &event_generation, type);
+	if (ret == -1)
+		/* very first transaction on this inode */
+		goto refresh;
+
+	if (local->event_generation != event_generation)
+		/* servers have disconnected / reconnected, and possibly
+		   rebooted, very likely changing the state of freshness
+		   of copies */
+		goto refresh;
+
+	read_subvol = afr_read_subvol_select_by_policy (inode, this,
+							local->readable);
+
+	if (read_subvol < 0 || read_subvol > priv->child_count) {
+		gf_log (this->name, GF_LOG_WARNING, "Unreadable subvolume %d "
+			"found with event generation %d", read_subvol,
+			event_generation);
+		goto refresh;
+	}
+
+	if (!local->child_up[read_subvol]) {
+		/* should never happen, just in case */
+		gf_log (this->name, GF_LOG_WARNING, "subvolume %d is the "
+			"read subvolume in this generation, but is not up",
+			read_subvol);
+		goto refresh;
+	}
+
+	local->read_attempted[read_subvol] = 1;
+
+	local->readfn (frame, this, read_subvol);
+
+	return 0;
+
+refresh:
+	afr_inode_refresh (frame, this, inode, afr_read_txn_refresh_done);
+
+	return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.c b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
deleted file mode 100644
index 629822a8e..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.c
+++ /dev/null
@@ -1,790 +0,0 @@
-/*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#include "glusterfs.h"
-#include "afr.h"
-#include "xlator.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "md5.h"
-
-#include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
-
-/*
-  This file contains the various self-heal algorithms
-*/
-
-static int
-sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
-                gf_boolean_t is_first_call, call_frame_t *old_loop_frame);
-static int
-sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame,
-                int32_t op_ret, int32_t op_errno);
-static int
-sh_destroy_frame (call_frame_t *frame, xlator_t *this)
-{
-        if (!frame)
-                goto out;
-
-        AFR_STACK_DESTROY (frame);
-out:
-        return 0;
-}
-
-static void
-sh_private_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t             *local   = NULL;
-        afr_self_heal_t         *sh      = NULL;
-        afr_sh_algo_private_t   *sh_priv = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = sh->private;
-        if (sh_priv)
-                GF_FREE (sh_priv);
-}
-
-static int
-sh_number_of_writes_needed (unsigned char *write_needed, int child_count)
-{
-        int writes = 0;
-        int i      = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (write_needed[i])
-                        writes++;
-        }
-
-        return writes;
-}
-
-
-static int
-sh_loop_driver_done (call_frame_t *sh_frame, xlator_t *this,
-                     call_frame_t *last_loop_frame)
-{
-        afr_local_t             *local        = NULL;
-        afr_self_heal_t         *sh           = NULL;
-        afr_sh_algo_private_t   *sh_priv      = NULL;
-        int32_t                 total_blocks = 0;
-        int32_t                 diff_blocks  = 0;
-
-        local        = sh_frame->local;
-        sh           = &local->self_heal;
-        sh_priv      = sh->private;
-        if (sh_priv) {
-                total_blocks = sh_priv->total_blocks;
-                diff_blocks  = sh_priv->diff_blocks;
-        }
-
-        sh_private_cleanup (sh_frame, this);
-        if (sh->op_failed) {
-                GF_ASSERT (!last_loop_frame);
-                //loop_finish should have happened and the old_loop should be NULL
-                gf_log (this->name, GF_LOG_INFO,
-                        "self-heal aborting on %s",
-                        local->loc.path);
-
-                local->self_heal.algo_abort_cbk (sh_frame, this);
-        } else {
-                GF_ASSERT (last_loop_frame);
-                if (diff_blocks == total_blocks) {
-                        gf_log (this->name, GF_LOG_INFO, "full self-heal "
-                                "completed on %s",local->loc.path);
-                } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "diff self-heal on %s: completed. "
-                                "(%d blocks of %d were different (%.2f%%))",
-                                local->loc.path, diff_blocks, total_blocks,
-                                ((diff_blocks * 1.0)/total_blocks) * 100);
-                }
-
-                sh->old_loop_frame = last_loop_frame;
-                local->self_heal.algo_completion_cbk (sh_frame, this);
-        }
-
-        return 0;
-}
-
-int
-sh_loop_finish (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_local_t             *loop_local = NULL;
-        afr_self_heal_t         *loop_sh = NULL;
-
-        if (!loop_frame)
-                goto out;
-
-        loop_local = loop_frame->local;
-        if (loop_local) {
-                loop_sh = &loop_local->self_heal;
-        }
-
-        if (loop_sh && loop_sh->data_lock_held) {
-                afr_sh_data_unlock (loop_frame, this,
-                                    sh_destroy_frame);
-        } else {
-                sh_destroy_frame (loop_frame, this);
-        }
-out:
-        return 0;
-}
-
-static int
-sh_loop_lock_success (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_local_t                 *loop_local = NULL;
-        afr_self_heal_t             *loop_sh    = NULL;
-
-        loop_local = loop_frame->local;
-        loop_sh = &loop_local->self_heal;
-
-        sh_loop_finish (loop_sh->old_loop_frame, this);
-        loop_sh->old_loop_frame = NULL;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Aquired lock for range %"PRIu64
-                " %"PRIu64, loop_sh->offset, loop_sh->block_size);
-        loop_sh->data_lock_held = _gf_true;
-        loop_sh->sh_data_algo_start (loop_frame, this);
-        return 0;
-}
-
-static int
-sh_loop_lock_failure (call_frame_t *loop_frame, xlator_t *this)
-{
-        call_frame_t                *sh_frame = NULL;
-        afr_local_t                 *loop_local = NULL;
-        afr_self_heal_t             *loop_sh    = NULL;
-
-        loop_local = loop_frame->local;
-        loop_sh = &loop_local->self_heal;
-        sh_frame = loop_sh->sh_frame;
-
-        gf_log (this->name, GF_LOG_ERROR, "failed lock for range %"PRIu64
-                " %"PRIu64, loop_sh->offset, loop_sh->block_size);
-        sh_loop_finish (loop_sh->old_loop_frame, this);
-        loop_sh->old_loop_frame = NULL;
-        sh_loop_return (sh_frame, this, loop_frame, -1, ENOTCONN);
-        return 0;
-}
-
-static int
-sh_loop_frame_create (call_frame_t *sh_frame, xlator_t *this,
-                      call_frame_t *old_loop_frame, call_frame_t **loop_frame)
-{
-        call_frame_t                *new_loop_frame = NULL;
-        afr_local_t                 *local          = NULL;
-        afr_self_heal_t             *sh             = NULL;
-        afr_local_t                 *new_loop_local = NULL;
-        afr_self_heal_t             *new_loop_sh    = NULL;
-        afr_private_t               *priv           = NULL;
-
-        GF_ASSERT (sh_frame);
-        GF_ASSERT (loop_frame);
-
-        *loop_frame = NULL;
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-        priv    = this->private;
-
-        new_loop_frame = copy_frame (sh_frame);
-        if (!new_loop_frame)
-                goto out;
-        //We want the frame to have same lk_owner as sh_frame
-        //so that locks translator allows conflicting locks
-        new_loop_local = afr_local_copy (local, this);
-        if (!new_loop_local)
-                goto out;
-        new_loop_frame->local = new_loop_local;
-
-        new_loop_sh = &new_loop_local->self_heal;
-        new_loop_sh->sources = memdup (sh->sources,
-                                       priv->child_count * sizeof (*sh->sources));
-        if (!new_loop_sh->sources)
-                goto out;
-        new_loop_sh->write_needed = GF_CALLOC (priv->child_count,
-                                               sizeof (*new_loop_sh->write_needed),
-                                               gf_afr_mt_char);
-        if (!new_loop_sh->write_needed)
-                goto out;
-        new_loop_sh->checksum = GF_CALLOC (priv->child_count, MD5_DIGEST_LEN,
-                                           gf_afr_mt_uint8_t);
-        if (!new_loop_sh->checksum)
-                goto out;
-        new_loop_sh->inode      = inode_ref (sh->inode);
-        new_loop_sh->sh_data_algo_start = sh->sh_data_algo_start;
-        new_loop_sh->source = sh->source;
-        new_loop_sh->active_sinks = sh->active_sinks;
-        new_loop_sh->healing_fd = fd_ref (sh->healing_fd);
-        new_loop_sh->file_has_holes = sh->file_has_holes;
-        new_loop_sh->old_loop_frame = old_loop_frame;
-        new_loop_sh->sh_frame = sh_frame;
-        *loop_frame = new_loop_frame;
-        return 0;
-out:
-        sh_destroy_frame (new_loop_frame, this);
-        return -ENOMEM;
-}
-
-static int
-sh_loop_start (call_frame_t *sh_frame, xlator_t *this, off_t offset,
-               call_frame_t *old_loop_frame)
-{
-        call_frame_t                *new_loop_frame = NULL;
-        afr_local_t                 *local          = NULL;
-        afr_self_heal_t             *sh             = NULL;
-        afr_local_t                 *new_loop_local = NULL;
-        afr_self_heal_t             *new_loop_sh    = NULL;
-        int                         ret             = 0;
-
-        GF_ASSERT (sh_frame);
-
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-
-        ret = sh_loop_frame_create (sh_frame, this, old_loop_frame,
-                                    &new_loop_frame);
-        if (ret)
-                goto out;
-        new_loop_local = new_loop_frame->local;
-        new_loop_sh = &new_loop_local->self_heal;
-        new_loop_sh->offset = offset;
-        new_loop_sh->block_size = sh->block_size;
-        afr_sh_data_lock (new_loop_frame, this, offset, new_loop_sh->block_size,
-                          sh_loop_lock_success, sh_loop_lock_failure);
-        return 0;
-out:
-        sh->op_failed = 1;
-        if (old_loop_frame)
-                sh_loop_finish (old_loop_frame, this);
-        sh_loop_return (sh_frame, this, new_loop_frame, -1, ENOMEM);
-        return 0;
-}
-
-static int
-sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
-                gf_boolean_t is_first_call, call_frame_t *old_loop_frame)
-{
-        afr_local_t *               local          = NULL;
-        afr_self_heal_t *           sh             = NULL;
-        afr_sh_algo_private_t       *sh_priv        = NULL;
-        gf_boolean_t                is_driver_done = _gf_false;
-        blksize_t                   block_size     = 0;
-        int                         loop           = 0;
-        off_t                       offset         = 0;
-        afr_private_t               *priv          = NULL;
-
-        priv    = this->private;
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-        LOCK (&sh_priv->lock);
-        {
-                if (!is_first_call)
-                        sh_priv->loops_running--;
-                offset = sh_priv->offset;
-                block_size = sh->block_size;
-                while ((!sh->eof_reached) && (0 == sh->op_failed) &&
-                       (sh_priv->loops_running < priv->data_self_heal_window_size)
-                       && (sh_priv->offset < sh->file_size)) {
-
-                        loop++;
-                        sh_priv->offset += block_size;
-                        sh_priv->loops_running++;
-
-                        if (!is_first_call)
-                                break;
-                }
-                if (0 == sh_priv->loops_running) {
-                        is_driver_done = _gf_true;
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        if (0 == loop) {
-                //loop finish does unlock, but the erasing of the pending
-                //xattrs needs to happen before that so do not finish the loop
-                if (is_driver_done && !sh->op_failed)
-                        goto driver_done;
-                if (old_loop_frame) {
-                        sh_loop_finish (old_loop_frame, this);
-                        old_loop_frame = NULL;
-                }
-        }
-
-        //If we have more loops to form we should finish previous loop after
-        //the next loop lock
-        while (loop--) {
-                if (sh->op_failed) {
-                        // op failed in other loop, stop spawning more loops
-                        if (old_loop_frame) {
-                                sh_loop_finish (old_loop_frame, this);
-                                old_loop_frame = NULL;
-                        }
-                        sh_loop_driver (sh_frame, this, _gf_false, NULL);
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE, "spawning a loop "
-                                "for offset %"PRId64, offset);
-
-                        sh_loop_start (sh_frame, this, offset, old_loop_frame);
-                        old_loop_frame = NULL;
-                        offset += block_size;
-                }
-        }
-
-driver_done:
-        if (is_driver_done) {
-                sh_loop_driver_done (sh_frame, this, old_loop_frame);
-        }
-        return 0;
-}
-
-static int
-sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame,
-                int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t *               loop_local = NULL;
-        afr_self_heal_t *           loop_sh    = NULL;
-        afr_local_t *               sh_local = NULL;
-        afr_self_heal_t            *sh       = NULL;
-
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        if (loop_frame) {
-                loop_local = loop_frame->local;
-                if (loop_local)
-                        loop_sh    = &loop_local->self_heal;
-                if (loop_sh)
-                        gf_log (this->name, GF_LOG_TRACE, "loop for offset "
-                                "%"PRId64" returned", loop_sh->offset);
-        }
-
-        if (op_ret == -1) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                if (loop_frame) {
-                        sh_loop_finish (loop_frame, this);
-                        loop_frame = NULL;
-                }
-        }
-
-        sh_loop_driver (sh_frame, this, _gf_false, loop_frame);
-
-        return 0;
-}
-
-static int
-sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *postbuf)
-{
-        afr_private_t *             priv        = NULL;
-        afr_local_t *               loop_local    = NULL;
-        afr_self_heal_t *           loop_sh       = NULL;
-        call_frame_t               *sh_frame    = NULL;
-        afr_local_t *               sh_local    = NULL;
-        afr_self_heal_t            *sh          = NULL;
-        int                         call_count  = 0;
-        int                         child_index = 0;
-
-        priv     = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        child_index =  (long) cookie;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "wrote %d bytes of data from %s to child %d, offset %"PRId64"",
-                op_ret, sh_local->loc.path, child_index, loop_sh->offset);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "write to %s failed on subvolume %s (%s)",
-                        sh_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-
-                sh->op_failed = 1;
-                afr_sh_set_error (loop_sh, op_errno);
-        }
-
-        call_count = afr_frame_return (loop_frame);
-
-        if (call_count == 0) {
-                sh_loop_return (sh_frame, this, loop_frame,
-                                loop_sh->op_ret, loop_sh->op_errno);
-        }
-
-        return 0;
-}
-
-
-static int
-sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref)
-{
-        afr_private_t *               priv       = NULL;
-        afr_local_t *                 loop_local   = NULL;
-        afr_self_heal_t *             loop_sh      = NULL;
-        call_frame_t                 *sh_frame   = NULL;
-        int                           i          = 0;
-        int                           call_count = 0;
-        afr_local_t *                 sh_local   = NULL;
-        afr_self_heal_t *             sh      = NULL;
-
-        priv       = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "read %d bytes of data from %s, offset %"PRId64"",
-                op_ret, loop_local->loc.path, loop_sh->offset);
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        sh->op_failed = 1;
-                        gf_log (this->name, GF_LOG_ERROR, "read failed on %d "
-                                "for %s reason :%s", sh->source,
-                                sh_local->loc.path, strerror (errno));
-                } else {
-                        sh->eof_reached = _gf_true;
-                        gf_log (this->name, GF_LOG_DEBUG, "Eof reached for %s",
-                                sh_local->loc.path);
-                }
-                sh_loop_return (sh_frame, this, loop_frame, op_ret, op_errno);
-                goto out;
-        }
-
-        if (loop_sh->file_has_holes && iov_0filled (vector, count) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG, "0 filled block");
-                        sh_loop_return (sh_frame, this, loop_frame,
-                                        op_ret, op_errno);
-                        goto out;
-        }
-
-        call_count = sh_number_of_writes_needed (loop_sh->write_needed,
-                                                 priv->child_count);
-        GF_ASSERT (call_count > 0);
-        loop_local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!loop_sh->write_needed[i])
-                        continue;
-                STACK_WIND_COOKIE (loop_frame, sh_loop_write_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->writev,
-                                   loop_sh->healing_fd, vector, count,
-                                   loop_sh->offset, 0, iobref);
-
-                if (!--call_count)
-                        break;
-        }
-
-out:
-        return 0;
-}
-
-
-static int
-sh_loop_read (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv       = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-
-        priv     = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        STACK_WIND_COOKIE (loop_frame, sh_loop_read_cbk,
-                           (void *) (long) loop_sh->source,
-                           priv->children[loop_sh->source],
-                           priv->children[loop_sh->source]->fops->readv,
-                           loop_sh->healing_fd, loop_sh->block_size,
-                           loop_sh->offset, 0);
-
-        return 0;
-}
-
-
-static int
-sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
-{
-        afr_private_t                 *priv         = NULL;
-        afr_local_t                   *loop_local   = NULL;
-        afr_self_heal_t               *loop_sh      = NULL;
-        call_frame_t                  *sh_frame     = NULL;
-        afr_local_t                   *sh_local     = NULL;
-        afr_self_heal_t               *sh           = NULL;
-        afr_sh_algo_private_t         *sh_priv      = NULL;
-        int                           child_index  = 0;
-        int                           call_count   = 0;
-        int                           i            = 0;
-        int                           write_needed = 0;
-
-        priv  = this->private;
-
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        sh_priv = sh->private;
-
-        child_index = (long) cookie;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "checksum on %s failed on subvolume %s (%s)",
-                        sh_local->loc.path, priv->children[child_index]->name,
-                        strerror (op_errno));
-                sh->op_failed = 1;
-        } else {
-                memcpy (loop_sh->checksum + child_index * MD5_DIGEST_LEN,
-                        strong_checksum, MD5_DIGEST_LEN);
-        }
-
-        call_count = afr_frame_return (loop_frame);
-
-        if (call_count == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sh->sources[i] || !sh_local->child_up[i])
-                                continue;
-
-                        if (memcmp (loop_sh->checksum + (i * MD5_DIGEST_LEN),
-                                    loop_sh->checksum + (sh->source * MD5_DIGEST_LEN),
-                                    MD5_DIGEST_LEN)) {
-                                /*
-                                  Checksums differ, so this block
-                                  must be written to this sink
-                                */
-
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "checksum on subvolume %s at offset %"
-                                        PRId64" differs from that on source",
-                                        priv->children[i]->name, loop_sh->offset);
-
-                                write_needed = loop_sh->write_needed[i] = 1;
-                        }
-                }
-
-                LOCK (&sh_priv->lock);
-                {
-                        sh_priv->total_blocks++;
-                        if (write_needed)
-                                sh_priv->diff_blocks++;
-                }
-                UNLOCK (&sh_priv->lock);
-
-                if (write_needed && !sh->op_failed) {
-                        sh_loop_read (loop_frame, this);
-                } else {
-                        sh_loop_return (sh_frame, this, loop_frame,
-                                        op_ret, op_errno);
-                }
-        }
-
-        return 0;
-}
-
-static int
-sh_diff_checksum (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv         = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-        int                     call_count    = 0;
-        int                     i             = 0;
-
-        priv         = this->private;
-        loop_local   = loop_frame->local;
-        loop_sh      = &loop_local->self_heal;
-
-        call_count = loop_sh->active_sinks + 1;  /* sinks and source */
-
-        loop_local->call_count = call_count;
-
-        STACK_WIND_COOKIE (loop_frame, sh_diff_checksum_cbk,
-                           (void *) (long) loop_sh->source,
-                           priv->children[loop_sh->source],
-                           priv->children[loop_sh->source]->fops->rchecksum,
-                           loop_sh->healing_fd,
-                           loop_sh->offset, loop_sh->block_size);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (loop_sh->sources[i] || !loop_local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (loop_frame, sh_diff_checksum_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->rchecksum,
-                                   loop_sh->healing_fd,
-                                   loop_sh->offset, loop_sh->block_size);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
-static int
-sh_full_read_write_to_sinks (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv         = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-        int                     i             = 0;
-
-        priv         = this->private;
-        loop_local   = loop_frame->local;
-        loop_sh      = &loop_local->self_heal;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (loop_sh->sources[i] || !loop_local->child_up[i])
-                        continue;
-                loop_sh->write_needed[i] = 1;
-        }
-        sh_loop_read (loop_frame, this);
-        return 0;
-}
-
-afr_sh_algo_private_t*
-afr_sh_priv_init ()
-{
-        afr_sh_algo_private_t   *sh_priv = NULL;
-
-        sh_priv = GF_CALLOC (1, sizeof (*sh_priv),
-                             gf_afr_mt_afr_private_t);
-        if (!sh_priv)
-                goto out;
-
-        LOCK_INIT (&sh_priv->lock);
-out:
-        return sh_priv;
-}
-
-void
-afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src,
-                      unsigned int child_count)
-{
-        afr_local_t             *dst_local   = NULL;
-        afr_self_heal_t         *dst_sh      = NULL;
-        afr_local_t             *src_local   = NULL;
-        afr_self_heal_t         *src_sh      = NULL;
-
-        dst_local = dst->local;
-        dst_sh = &dst_local->self_heal;
-        src_local = src->local;
-        src_sh = &src_local->self_heal;
-        GF_ASSERT (src_sh->data_lock_held);
-        GF_ASSERT (!dst_sh->data_lock_held);
-        afr_lk_transfer_datalock (dst, src, child_count);
-        src_sh->data_lock_held = _gf_false;
-        dst_sh->data_lock_held = _gf_true;
-}
-
-int
-afr_sh_start_loops (call_frame_t *sh_frame, xlator_t *this,
-                    afr_sh_algo_fn sh_data_algo_start)
-{
-        call_frame_t            *first_loop_frame = NULL;
-        afr_local_t             *local   = NULL;
-        afr_self_heal_t         *sh      = NULL;
-        int                     ret      = 0;
-        afr_private_t           *priv    = NULL;
-
-        local = sh_frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        sh->sh_data_algo_start = sh_data_algo_start;
-        local->call_count = 0;
-        ret = sh_loop_frame_create (sh_frame, this, NULL, &first_loop_frame);
-        if (ret)
-                goto out;
-        afr_sh_transfer_lock (first_loop_frame, sh_frame, priv->child_count);
-        sh->private = afr_sh_priv_init ();
-        if (!sh->private) {
-                ret = -1;
-                goto out;
-        }
-        sh_loop_driver (sh_frame, this, _gf_true, first_loop_frame);
-        ret = 0;
-out:
-        if (ret) {
-                sh->op_failed = 1;
-                sh_loop_driver_done (sh_frame, this, NULL);
-        }
-        return 0;
-}
-
-int
-afr_sh_algo_diff (call_frame_t *sh_frame, xlator_t *this)
-{
-        afr_sh_start_loops (sh_frame, this, sh_diff_checksum);
-        return 0;
-}
-
-int
-afr_sh_algo_full (call_frame_t *sh_frame, xlator_t *this)
-{
-        afr_sh_start_loops (sh_frame, this, sh_full_read_write_to_sinks);
-        return 0;
-}
-
-struct afr_sh_algorithm afr_self_heal_algorithms[] = {
-        {.name = "full",  .fn = afr_sh_algo_full},
-        {.name = "diff",  .fn = afr_sh_algo_diff},
-        {0, 0},
-};
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.h b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
deleted file mode 100644
index 04d8e8a6c..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __AFR_SELF_HEAL_ALGORITHM_H__
-#define __AFR_SELF_HEAL_ALGORITHM_H__
-
-
-typedef int (*afr_sh_algo_fn) (call_frame_t *frame,
-                               xlator_t *this);
-
-struct afr_sh_algorithm {
-        const char *name;
-        afr_sh_algo_fn fn;
-};
-
-extern struct afr_sh_algorithm afr_self_heal_algorithms[3];
-typedef struct {
-        gf_lock_t lock;
-        unsigned int loops_running;
-        off_t offset;
-
-        int32_t total_blocks;
-        int32_t diff_blocks;
-} afr_sh_algo_private_t;
-
-#endif /* __AFR_SELF_HEAL_ALGORITHM_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index f4f74940c..4dac83113 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1,2287 +1,1009 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "byte-order.h"
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
 
 #include "afr.h"
-#include "afr-transaction.h"
-#include "afr-self-heal-common.h"
 #include "afr-self-heal.h"
-#include "pump.h"
+#include "byte-order.h"
 
-void
-afr_sh_reset (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        memset (sh->child_errno, 0,
-                sizeof (*sh->child_errno) * priv->child_count);
-        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
-        memset (sh->parentbufs, 0,
-                sizeof (*sh->parentbufs) * priv->child_count);
-        memset (sh->success, 0, sizeof (*sh->success) * priv->child_count);
-        memset (sh->locked_nodes, 0,
-                sizeof (*sh->locked_nodes) * priv->child_count);
-        sh->active_sinks = 0;
-
-        afr_reset_xattr (sh->xattr, priv->child_count);
-}
 
-//Intersection[child]=1 if child is part of intersection
-void
-afr_children_intersection_get (int32_t *set1, int32_t *set2,
-                               int *intersection, unsigned int child_count)
+int
+afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int                      i = 0;
-
-        memset (intersection, 0, sizeof (*intersection) * child_count);
-        for (i = 0; i < child_count; i++) {
-                intersection[i] = afr_is_child_present (set1, child_count, i)
-                                     && afr_is_child_present (set2, child_count,
-                                                              i);
-        }
-}
+	afr_local_t *local = NULL;
 
-/**
- * select_source - select a source and return it
- */
+	local = frame->local;
 
-int
-afr_sh_select_source (int sources[], int child_count)
-{
-        int i = 0;
-        for (i = 0; i < child_count; i++)
-                if (sources[i])
-                        return i;
+	syncbarrier_wake (&local->barrier);
 
-        return -1;
+	return 0;
 }
 
-void
-afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this)
-{
-        int              i = 0;
-        afr_local_t     *local      = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              active_sinks = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-                        active_sinks++;
-                        sh->success[i] = 1;
-                } else if (sh->sources[i] == 1 && local->child_up[i] == 1) {
-                        sh->success[i] = 1;
-                }
-        }
-        sh->active_sinks = active_sinks;
-}
 
 int
-afr_sh_source_count (int sources[], int child_count)
+afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      int subvol, dict_t *xattr)
 {
-        int i = 0;
-        int nsource = 0;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	loc_t loc = {0, };
 
-        for (i = 0; i < child_count; i++)
-                if (sources[i])
-                        nsource++;
-        return nsource;
-}
+	priv = this->private;
+	local = frame->local;
 
-void
-afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno)
-{
-        sh->op_ret = -1;
-        if (afr_error_more_important (sh->op_errno, op_errno))
-                sh->op_errno = op_errno;
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
-{
-        afr_private_t *  priv = this->private;
-        char            *buf  = NULL;
-        char            *ptr  = NULL;
-        int              i    = 0;
-        int              j    = 0;
-
-        /* 10 digits per entry + 1 space + '[' and ']' */
-        buf = GF_MALLOC (priv->child_count * 11 + 8, gf_afr_mt_char);
-
-        for (i = 0; i < priv->child_count; i++) {
-                ptr = buf;
-                ptr += sprintf (ptr, "[ ");
-                for (j = 0; j < priv->child_count; j++) {
-                        ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
-                }
-                sprintf (ptr, "]");
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pending_matrix: %s", buf);
-        }
-
-        GF_FREE (buf);
+	STACK_WIND (frame, afr_selfheal_post_op_cbk, priv->children[subvol],
+		    priv->children[subvol]->fops->xattrop, &loc,
+		    GF_XATTROP_ADD_ARRAY, xattr, NULL);
+
+	syncbarrier_wait (&local->barrier, 1);
+
+	return 0;
 }
 
-void
-afr_init_pending_matrix (int32_t **pending_matrix, size_t child_count)
+
+dict_t *
+afr_selfheal_output_xattr (xlator_t *this, afr_transaction_type type,
+			   int *output_dirty, int **output_matrix, int subvol)
 {
-        int             i   = 0;
-        int             j   = 0;
+	dict_t *xattr = NULL;
+	afr_private_t *priv = NULL;
+	int j = 0;
+	int idx = 0;
+	int ret = 0;
+	int *raw = 0;
 
-        GF_ASSERT (pending_matrix);
+	priv = this->private;
+	idx = afr_index_for_transaction_type (type);
 
-        for (i = 0; i < child_count; i++) {
-                for (j = 0; j < child_count; j++) {
-                        pending_matrix[i][j] = 0;
-                }
-        }
-}
+	xattr = dict_new ();
+	if (!xattr)
+		return NULL;
 
-void
-afr_mark_ignorant_subvols_as_pending (int32_t **pending_matrix,
-                                      unsigned char *ignorant_subvols,
-                                      size_t  child_count)
-{
-        int            i                = 0;
-        int            j                = 0;
-
-        GF_ASSERT (pending_matrix);
-        GF_ASSERT (ignorant_subvols);
-
-        for (i = 0; i < child_count; i++) {
-                if (ignorant_subvols[i]) {
-                        for (j = 0; j < child_count; j++) {
-                                if (!ignorant_subvols[j])
-                                        pending_matrix[j][i] += 1;
-                        }
-                }
-        }
+	if (output_dirty[subvol]) {
+		/* clear dirty */
+		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+		if (!raw)
+			goto err;
+
+		raw[idx] = hton32 (output_dirty[subvol]);
+		ret = dict_set_bin (xattr, AFR_DIRTY, raw,
+				    sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret)
+			goto err;
+	}
+
+	/* clear/set pending */
+	for (j = 0; j < priv->child_count; j++) {
+		if (!output_matrix[subvol][j])
+			continue;
+
+		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS,
+				 gf_afr_mt_int32_t);
+		if (!raw)
+			goto err;
+
+		raw[idx] = hton32 (output_matrix[subvol][j]);
+
+		ret = dict_set_bin (xattr, priv->pending_key[j],
+				    raw, sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret)
+			goto err;
+	}
+
+	return xattr;
+err:
+	if (xattr)
+		dict_unref (xattr);
+	return NULL;
 }
 
+
 int
-afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
-                          unsigned char *ignorant_subvols,
-                          dict_t *xattr[], afr_transaction_type type,
-                          size_t child_count)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]       = {0,};
-        void          *pending_raw      = NULL;
-        int            ret              = -1;
-        int            i                = 0;
-        int            j                = 0;
-        int            k                = 0;
-
-        afr_init_pending_matrix (pending_matrix, child_count);
-
-        for (i = 0; i < child_count; i++) {
-                pending_raw = NULL;
-
-                for (j = 0; j < child_count; j++) {
-                        ret = dict_get_ptr (xattr[i], pending_key[j],
-                                            &pending_raw);
-
-                        if (ret != 0) {
-                                /*
-                                 * There is no xattr present. This means this
-                                 * subvolume should be considered an 'ignorant'
-                                 * subvolume.
-                                 */
-
-                                if (ignorant_subvols)
-                                        ignorant_subvols[i] = 1;
-                                continue;
-                        }
-
-                        memcpy (pending, pending_raw, sizeof(pending));
-                        k = afr_index_for_transaction_type (type);
-
-                        pending_matrix[i][j] = ntoh32 (pending[k]);
-                }
-        }
-
-        return ret;
+afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			   unsigned char *sources, unsigned char *sinks,
+			   unsigned char *healed_sinks, afr_transaction_type type,
+			   struct afr_reply *replies, unsigned char *locked_on)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int j = 0;
+	unsigned char *pending = NULL;
+	int *input_dirty = NULL;
+	int **input_matrix = NULL;
+	int *output_dirty = NULL;
+	int **output_matrix = NULL;
+	dict_t *xattr = NULL;
+
+	priv = this->private;
+
+	pending = alloca0 (priv->child_count);
+
+	input_dirty = alloca0 (priv->child_count * sizeof (int));
+	input_matrix = ALLOC_MATRIX (priv->child_count, int);
+	output_dirty = alloca0 (priv->child_count * sizeof (int));
+	output_matrix = ALLOC_MATRIX (priv->child_count, int);
+
+	afr_selfheal_extract_xattr (this, replies, type, input_dirty,
+				    input_matrix);
+
+	for (i = 0; i < priv->child_count; i++)
+		if (sinks[i] && !healed_sinks[i])
+			pending[i] = 1;
+
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			if (pending[j])
+				output_matrix[i][j] = 1;
+			else
+				output_matrix[i][j] = -input_matrix[i][j];
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!pending[i])
+			output_dirty[i] = -input_dirty[i];
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!locked_on[i])
+			/* perform post-op only on subvols we had locked
+			   and inspected on.
+			*/
+			continue;
+
+		xattr = afr_selfheal_output_xattr (this, type, output_dirty,
+						   output_matrix, i);
+		if (!xattr) {
+			gf_log (this->name, GF_LOG_ERROR,
+				"unable to allocate xdata for subvol %d", i);
+			continue;
+		}
+
+		afr_selfheal_post_op (frame, this, inode, i, xattr);
+
+		dict_unref (xattr);
+	}
+
+	return 0;
 }
 
-typedef enum {
-        AFR_NODE_INVALID,
-        AFR_NODE_INNOCENT,
-        AFR_NODE_FOOL,
-        AFR_NODE_WISE,
-} afr_node_type;
 
-typedef struct {
-        afr_node_type type;
-        int           wisdom;
-} afr_node_character;
+void
+afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count)
+{
+	int i = 0;
+	dict_t *xdata = NULL;
+
+	if (dst == src)
+		return;
+
+	for (i = 0; i < count; i++) {
+		dst[i].valid = src[i].valid;
+		dst[i].op_ret = src[i].op_ret;
+		dst[i].op_errno = src[i].op_errno;
+		dst[i].prestat = src[i].prestat;
+		dst[i].poststat = src[i].poststat;
+		dst[i].preparent = src[i].preparent;
+		dst[i].postparent = src[i].postparent;
+		dst[i].preparent2 = src[i].preparent2;
+		dst[i].postparent2 = src[i].postparent2;
+		if (src[i].xdata)
+			xdata = dict_ref (src[i].xdata);
+		else
+			xdata = NULL;
+		if (dst[i].xdata)
+			dict_unref (dst[i].xdata);
+		dst[i].xdata = xdata;
+		memcpy (dst[i].checksum, src[i].checksum,
+			MD5_DIGEST_LENGTH);
+	}
+}
 
 
-static int
-afr_sh_is_innocent (int32_t *array, int child_count)
+int
+afr_selfheal_fill_dirty (xlator_t *this, int *dirty, int subvol,
+			 int idx, dict_t *xdata)
 {
-        int i   = 0;
-        int ret = 1;   /* innocent until proven guilty */
+	void *pending_raw = NULL;
+	int pending[3] = {0, };
 
-        for (i = 0; i < child_count; i++) {
-                if (array[i]) {
-                        ret = 0;
-                        break;
-                }
-        }
+	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw))
+		return -1;
 
-        return ret;
-}
+	if (!pending_raw)
+		return -1;
 
+	memcpy (pending, pending_raw, sizeof(pending));
 
-static int
-afr_sh_is_fool (int32_t *array, int i, int child_count)
-{
-        return array[i];   /* fool if accuses itself */
+	dirty[subvol] = ntoh32 (pending[idx]);
+
+	return 0;
 }
 
 
-static int
-afr_sh_is_wise (int32_t *array, int i, int child_count)
+int
+afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol,
+			  int idx, dict_t *xdata)
 {
-        return !array[i];  /* wise if does not accuse itself */
-}
+	int i = 0;
+	void *pending_raw = NULL;
+	int pending[3] = {0, };
+	afr_private_t *priv = NULL;
 
+	priv = this->private;
 
-static int
-afr_sh_all_nodes_innocent (afr_node_character *characters,
-                           int child_count)
-{
-        int i   = 0;
-        int ret = 1;
+	for (i = 0; i < priv->child_count; i++) {
+		if (dict_get_ptr (xdata, priv->pending_key[i], &pending_raw))
+			continue;
+
+		if (!pending_raw)
+			continue;
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_INNOCENT) {
-                        ret = 0;
-                        break;
-                }
-        }
+		memcpy (pending, pending_raw, sizeof(pending));
 
-        return ret;
+		matrix[subvol][i] = ntoh32 (pending[idx]);
+	}
+
+	return 0;
 }
 
 
-static int
-afr_sh_wise_nodes_exist (afr_node_character *characters, int child_count)
+int
+afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
+			    afr_transaction_type type, int *dirty, int **matrix)
 {
-        int i   = 0;
-        int ret = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	dict_t *xdata = NULL;
+	int idx = -1;
+
+	idx = afr_index_for_transaction_type (type);
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        ret = 1;
-                        break;
-                }
-        }
+	priv = this->private;
 
-        return ret;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].xdata)
+			continue;
+
+		xdata = replies[i].xdata;
+
+		afr_selfheal_fill_dirty (this, dirty, i, idx, xdata);
+		afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
+	}
+
+	return 0;
 }
 
 
+
 /*
- * The 'wisdom' of a wise node is 0 if any other wise node accuses it.
- * It is 1 if no other wise node accuses it.
- * Only wise nodes with wisdom 1 are sources.
+ * This function determines if a self-heal is required for a given inode,
+ * and if needed, in what direction.
+ *
+ * locked_on[] is the array representing servers which have been locked and
+ * from which xattrs have been fetched for analysis.
+ *
+ * The output of the function is by filling the arrays sources[] and sinks[].
+ *
+ * sources[i] is set if i'th server is an eligible source for a selfheal.
+ *
+ * sinks[i] is set if i'th server needs to be healed.
+ *
+ * if sources[0..N] are all set, there is no need for a selfheal.
+ *
+ * if sinks[0..N] are all set, the inode is in split brain.
  *
- * If no nodes with wisdom 1 exist, a split-brain has occurred.
  */
 
-static void
-afr_sh_compute_wisdom (int32_t *pending_matrix[],
-                       afr_node_character characters[], int child_count)
-{
-        int i = 0;
-        int j = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        characters[i].wisdom = 1;
-
-                        for (j = 0; j < child_count; j++) {
-                                if ((characters[j].type == AFR_NODE_WISE)
-                                    && pending_matrix[j][i]) {
-
-                                        characters[i].wisdom = 0;
-                                }
-                        }
-                }
-        }
+int
+afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
+			     struct afr_reply *replies,
+			     afr_transaction_type type, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int j = 0;
+	int *dirty = NULL;
+	int **matrix = NULL;
+	char *accused = NULL;
+
+	priv = this->private;
+
+	dirty = alloca0 (priv->child_count * sizeof (int));
+	accused = alloca0 (priv->child_count);
+	matrix = ALLOC_MATRIX(priv->child_count, int);
+
+	/* First construct the pending matrix for further analysis */
+	afr_selfheal_extract_xattr (this, replies, type, dirty, matrix);
+
+	/* Next short list all accused to exclude them from being sources */
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			if (matrix[i][j])
+				accused[j] = 1;
+		}
+	}
+
+	/* Short list all non-accused as sources */
+	memset (sources, 0, priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!accused[i] && locked_on[i])
+			sources[i] = 1;
+	}
+
+	/* Everyone accused by sources are sinks */
+	memset (sinks, 0, priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		for (j = 0; j < priv->child_count; j++) {
+			if (matrix[i][j])
+				sinks[j] = 1;
+		}
+	}
+
+	/* If any source has 'dirty' bit, pick first
+	   'dirty' source and make everybody else sinks */
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i] && dirty[i]) {
+			for (j = 0; j < priv->child_count; j++) {
+				if (j != i) {
+					sources[j] = 0;
+					sinks[j] = 1;
+				}
+			}
+			break;
+		}
+	}
+
+	/* If no sources, all locked nodes are sinks - split brain */
+	if (AFR_COUNT (sources, priv->child_count) == 0) {
+		for (i = 0; i < priv->child_count; i++) {
+			if (locked_on[i])
+				sinks[i] = 1;
+		}
+	}
+
+	return 0;
 }
 
 
-static int
-afr_sh_wise_nodes_conflict (afr_node_character *characters,
-                            int child_count)
+int
+afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			   int op_ret, int op_errno, inode_t *inode,
+			   struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
 {
-        int i   = 0;
-        int ret = 1;
+	afr_local_t *local = NULL;
+	int i = -1;
 
-        for (i = 0; i < child_count; i++) {
-                if ((characters[i].type == AFR_NODE_WISE)
-                    && characters[i].wisdom == 1) {
+	local = frame->local;
+	i = (long) cookie;
 
-                        /* There is atleast one bona-fide wise node */
-                        ret = 0;
-                        break;
-                }
-        }
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (buf)
+		local->replies[i].poststat = *buf;
+	if (parbuf)
+		local->replies[i].postparent = *parbuf;
+	if (xdata)
+		local->replies[i].xdata = dict_ref (xdata);
 
-        return ret;
+	syncbarrier_wake (&local->barrier);
+
+	return 0;
 }
 
 
-static int
-afr_sh_mark_wisest_as_sources (int sources[],
-                               afr_node_character *characters,
-                               int child_count)
+inode_t *
+afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
+				 const char *name, struct afr_reply *replies,
+				 unsigned char *lookup_on)
 {
-        int nsources = 0;
-        int i        = 0;
+	loc_t loc = {0, };
+	dict_t *xattr_req = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	inode_t *inode = NULL;
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].wisdom == 1) {
-                        sources[i] = 1;
-                        nsources++;
-                }
-        }
+	local = frame->local;
+	priv = frame->this->private;
 
-        return nsources;
-}
+	xattr_req = dict_new ();
+	if (!xattr_req)
+		return NULL;
 
-static void
-afr_compute_witness_of_fools (int32_t *witnesses, int32_t **pending_matrix,
-                              afr_node_character *characters,
-                              int32_t child_count)
-{
-        int i       = 0;
-        int j       = 0;
-        int witness = 0;
-
-        GF_ASSERT (witnesses);
-        GF_ASSERT (pending_matrix);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
-
-                witness = 0;
-                for (j = 0; j < child_count; j++) {
-                        if (i == j)
-                                continue;
-                        witness += pending_matrix[i][j];
-                }
-                witnesses[i] = witness;
-        }
-}
+	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
+		dict_destroy (xattr_req);
+		return NULL;
+	}
 
-static int32_t
-afr_find_biggest_witness_among_fools (int32_t *witnesses,
-                                      afr_node_character *characters,
-                                      int32_t child_count)
-{
-        int i               = 0;
-        int biggest_witness = -1;
+	inode = inode_new (parent->table);
+	if (!inode) {
+		dict_destroy (xattr_req);
+		return NULL;
+	}
 
-        GF_ASSERT (witnesses);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
+	loc.parent = inode_ref (parent);
+	uuid_copy (loc.pargfid, parent->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
+	AFR_ONLIST (lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+		    xattr_req);
 
-                if (biggest_witness < witnesses[i])
-                        biggest_witness = witnesses[i];
-        }
-        return biggest_witness;
-}
+	afr_replies_copy (replies, local->replies, priv->child_count);
 
-int
-afr_mark_fool_as_source_by_witness (int32_t *sources, int32_t *witnesses,
-                                    afr_node_character *characters,
-                                    int32_t child_count, int32_t witness)
-{
-        int i        = 0;
-        int nsources = 0;
-
-        GF_ASSERT (sources);
-        GF_ASSERT (witnesses);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
-
-                if (witness == witnesses[i]) {
-                        sources[i] = 1;
-                        nsources++;
-                }
-        }
-        return nsources;
-}
+	loc_wipe (&loc);
+	dict_unref (xattr_req);
 
-static int
-afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
-                                     afr_node_character *characters,
-                                     int child_count)
-{
-        int32_t       biggest_witness = 0;
-        int           nsources        = 0;
-        int32_t       *witnesses      = NULL;
-
-        GF_ASSERT (child_count > 0);
-
-        witnesses = GF_CALLOC (child_count, sizeof (*witnesses),
-                               gf_afr_mt_int32_t);
-        if (NULL == witnesses) {
-                nsources = -1;
-                goto out;
-        }
-
-        afr_compute_witness_of_fools (witnesses, pending_matrix, characters,
-                                      child_count);
-        biggest_witness = afr_find_biggest_witness_among_fools (witnesses,
-                                                                characters,
-                                                                child_count);
-        nsources = afr_mark_fool_as_source_by_witness (sources, witnesses,
-                                                       characters, child_count,
-                                                       biggest_witness);
-out:
-        if (witnesses)
-                GF_FREE (witnesses);
-        return nsources;
+	return inode;
 }
 
-int
-afr_mark_child_as_source_by_uid (int32_t *sources, struct iatt *bufs,
-                                 int32_t *success_children,
-                                 unsigned int child_count, uint32_t uid)
-{
-        int     i        = 0;
-        int     nsources = 0;
-        int     child    = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (-1 == success_children[i])
-                        break;
-
-                child = success_children[i];
-                if (uid == bufs[child].ia_uid) {
-                        sources[child] = 1;
-                        nsources++;
-                }
-        }
-        return nsources;
-}
 
 int
-afr_get_child_with_lowest_uid (struct iatt *bufs, int32_t *success_children,
-                               unsigned int child_count)
+afr_selfheal_unlocked_discover_on (call_frame_t *frame, inode_t *inode,
+				   uuid_t gfid, struct afr_reply *replies,
+				   unsigned char *discover_on)
 {
-        int     i        = 0;
-        int     smallest = -1;
-        int     child    = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (-1 == success_children[i])
-                        break;
-                child = success_children[i];
-                if ((smallest == -1) ||
-                    (bufs[child].ia_uid < bufs[smallest].ia_uid)) {
-                        smallest = child;
-                }
-        }
-        return smallest;
-}
+	loc_t loc = {0, };
+	dict_t *xattr_req = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
-static int
-afr_sh_mark_lowest_uid_as_source (struct iatt *bufs, int32_t *success_children,
-                                  int child_count, int32_t *sources)
-{
-        int   nsources              = 0;
-        int   smallest              = 0;
-
-        smallest = afr_get_child_with_lowest_uid (bufs, success_children,
-                                                  child_count);
-        if (smallest < 0) {
-                nsources = -1;
-                goto out;
-        }
-        nsources = afr_mark_child_as_source_by_uid (sources, bufs,
-                                                    success_children, child_count,
-                                                    bufs[smallest].ia_uid);
-out:
-        return nsources;
-}
+	local = frame->local;
+	priv = frame->this->private;
 
-char *
-afr_get_character_str (afr_node_type type)
-{
-        char *character = NULL;
-
-        switch (type) {
-        case AFR_NODE_INNOCENT:
-                character = "innocent";
-                break;
-        case AFR_NODE_FOOL:
-                character = "fool";
-                break;
-        case AFR_NODE_WISE:
-                character = "wise";
-                break;
-        default:
-                character = "invalid";
-                break;
-        }
-        return character;
-}
+	xattr_req = dict_new ();
+	if (!xattr_req)
+		return -ENOMEM;
 
-afr_node_type
-afr_find_child_character_type (int32_t *pending_row, int32_t child,
-                               unsigned int child_count)
-{
-        afr_node_type type = AFR_NODE_INVALID;
+	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
+		dict_destroy (xattr_req);
+		return -ENOMEM;
+	}
 
-        GF_ASSERT ((child >= 0) && (child < child_count));
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, gfid);
 
-        if (afr_sh_is_innocent (pending_row, child_count))
-                type = AFR_NODE_INNOCENT;
-        else if (afr_sh_is_fool (pending_row, child, child_count))
-                type = AFR_NODE_FOOL;
-        else if (afr_sh_is_wise (pending_row, child, child_count))
-                type = AFR_NODE_WISE;
-        return type;
-}
+	AFR_ONLIST (discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+		    xattr_req);
 
-int
-afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
-                   int32_t **pending_matrix, int32_t *sources,
-                   int32_t *success_children, afr_transaction_type type,
-                   int32_t *subvol_status, gf_boolean_t ignore_ignorant)
-{
-        afr_private_t           *priv = NULL;
-        afr_self_heal_type      sh_type    = AFR_SELF_HEAL_INVALID;
-        int                     nsources   = -1;
-        unsigned char           *ignorant_subvols = NULL;
-        unsigned int            child_count = 0;
-
-        priv = this->private;
-        child_count = priv->child_count;
-
-        if (afr_get_children_count (success_children, priv->child_count) == 0)
-                goto out;
-
-        if (!ignore_ignorant) {
-                ignorant_subvols = GF_CALLOC (sizeof (*ignorant_subvols),
-                                              child_count, gf_afr_mt_char);
-                if (NULL == ignorant_subvols)
-                        goto out;
-        }
-
-        afr_build_pending_matrix (priv->pending_key, pending_matrix,
-                                  ignorant_subvols, xattr, type,
-                                  priv->child_count);
-
-        if (!ignore_ignorant)
-                afr_mark_ignorant_subvols_as_pending (pending_matrix,
-                                                      ignorant_subvols,
-                                                      priv->child_count);
-        sh_type = afr_self_heal_type_for_transaction (type);
-        if (AFR_SELF_HEAL_INVALID == sh_type)
-                goto out;
-
-        afr_sh_print_pending_matrix (pending_matrix, this);
-
-        nsources = afr_mark_sources (this, sources, pending_matrix, bufs,
-                                     sh_type, success_children, subvol_status);
-out:
-        GF_FREE (ignorant_subvols);
-        return nsources;
-}
+	afr_replies_copy (replies, local->replies, priv->child_count);
 
-void
-afr_find_character_types (afr_node_character *characters,
-                          int32_t **pending_matrix, int32_t *success_children,
-                          unsigned int child_count)
-{
-        afr_node_type type  = AFR_NODE_INVALID;
-        int           child = 0;
-        int           i     = 0;
-
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child == -1)
-                        break;
-                type = afr_find_child_character_type (pending_matrix[child],
-                                                      child, child_count);
-                characters[child].type = type;
-        }
-}
+	loc_wipe (&loc);
+	dict_unref (xattr_req);
 
-void
-afr_mark_success_children_sources (int32_t *sources, int32_t *success_children,
-                                   unsigned int child_count)
-{
-        int i = 0;
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                sources[success_children[i]] = 1;
-        }
+	return 0;
 }
-/**
- * mark_sources: Mark all 'source' nodes and return number of source
- * nodes found
- *
- * A node (a row in the pending matrix) belongs to one of
- * three categories:
- *
- * M is the pending matrix.
- *
- * 'innocent' - M[i] is all zeroes
- * 'fool'     - M[i] has i'th element = 1 (self-reference)
- * 'wise'     - M[i] has i'th element = 0, others are 1 or 0.
- *
- * All 'innocent' nodes are sinks. If all nodes are innocent, no self-heal is
- * needed.
- *
- * A 'wise' node can be a source. If two 'wise' nodes conflict, it is
- * a split-brain. If one wise node refers to the other but the other doesn't
- * refer back, the referrer is a source.
- *
- * All fools are sinks, unless there are no 'wise' nodes. In that case,
- * one of the fools is made a source.
- */
 
 int
-afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
-                  struct iatt *bufs, afr_self_heal_type type,
-                  int32_t *success_children, int32_t *subvol_status)
+afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
+				uuid_t gfid, struct afr_reply *replies)
 {
-        /* stores the 'characters' (innocent, fool, wise) of the nodes */
-        afr_node_character *characters =  NULL;
-        int                nsources    = -1;
-        unsigned int       child_count = 0;
-        afr_private_t      *priv       = NULL;
-
-        priv = this->private;
-        child_count = priv->child_count;
-        characters = GF_CALLOC (sizeof (afr_node_character),
-                                child_count, gf_afr_mt_afr_node_character);
-        if (!characters)
-                goto out;
-
-        this = THIS;
-
-        /* start clean */
-        memset (sources, 0, sizeof (*sources) * child_count);
-        nsources = 0;
-        afr_find_character_types (characters, pending_matrix, success_children,
-                                  child_count);
-        if (afr_sh_all_nodes_innocent (characters, child_count)) {
-                if (type == AFR_SELF_HEAL_METADATA)
-                        nsources = afr_sh_mark_lowest_uid_as_source (bufs,
-                                                             success_children,
-                                                             child_count,
-                                                             sources);
-                goto out;
-        }
-
-        if (afr_sh_wise_nodes_exist (characters, child_count)) {
-                afr_sh_compute_wisdom (pending_matrix, characters, child_count);
-
-                if (afr_sh_wise_nodes_conflict (characters, child_count)) {
-                        if (subvol_status)
-                                *subvol_status |= SPLIT_BRAIN;
-                        nsources = -1;
-                } else {
-                        nsources = afr_sh_mark_wisest_as_sources (sources,
-                                                                  characters,
-                                                                  child_count);
-                }
-        } else {
-                if (subvol_status)
-                        *subvol_status |= ALL_FOOLS;
-                nsources = afr_mark_biggest_of_fools_as_source (sources,
-                                                                pending_matrix,
-                                                                characters,
-                                                                child_count);
-        }
+	afr_private_t *priv = NULL;
 
-out:
-        if (nsources == 0)
-                afr_mark_success_children_sources (sources, success_children,
-                                                   child_count);
-        GF_FREE (characters);
+	priv = frame->this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Number of sources: %d", nsources);
-        return nsources;
-}
-
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], unsigned char success[],
-                         int child_count, afr_transaction_type type)
-{
-        int i = 0;
-        int j = 0;
-
-        afr_build_pending_matrix (priv->pending_key, delta_matrix, NULL,
-                                  xattr, type, priv->child_count);
-        for (i = 0; i < priv->child_count; i++)
-                for (j = 0; j < priv->child_count; j++)
-                        delta_matrix[i][j] = -delta_matrix[i][j];
+	return afr_selfheal_unlocked_discover_on (frame, inode, gfid, replies,
+						  priv->child_up);
 }
 
 
 int
-afr_sh_delta_to_xattr (afr_private_t *priv,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-                       int child_count, afr_transaction_type type)
+afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int op_ret, int op_errno, dict_t *xdata)
 {
-        int      i       = 0;
-        int      j       = 0;
-        int      k       = 0;
-        int      ret     = 0;
-        int32_t *pending = NULL;
-
-        for (i = 0; i < child_count; i++) {
-                if (!xattr[i])
-                        continue;
-
-                for (j = 0; j < child_count; j++) {
-                        pending = GF_CALLOC (sizeof (int32_t), 3,
-                                             gf_afr_mt_int32_t);
-
-                        if (!pending)
-                                continue;
-                        /* 3 = data+metadata+entry */
-
-                        k = afr_index_for_transaction_type (type);
-
-                        pending[k] = hton32 (delta_matrix[i][j]);
-
-                        ret = dict_set_bin (xattr[i], priv->pending_key[j],
-                                            pending,
-                                            3 * sizeof (int32_t));
-                        if (ret < 0)
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-                }
-        }
-        return 0;
-}
+	afr_local_t *local = NULL;
+	int i = 0;
 
+	local = frame->local;
+	i = (long) cookie;
 
-int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
 
-        priv = this->private;
+	syncbarrier_wake (&local->barrier);
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+	return 0;
+}
 
-                if (ret != 0)
-                        return 0;
 
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
+int
+afr_selfheal_locked_fill (call_frame_t *frame, xlator_t *this,
+			  unsigned char *locked_on)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	int count = 0;
+
+	local = frame->local;
+	priv = this->private;
 
-                if (pending[j])
-                        return 1;
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+			locked_on[i] = 1;
+			count++;
+		} else {
+			locked_on[i] = 0;
+		}
+	}
 
-        return 0;
+	return count;
 }
 
 
 int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this)
+afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, off_t off, size_t size,
+			 unsigned char *locked_on)
 {
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
-
-        priv = this->private;
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-                if (ret != 0)
-                        return 0;
+	flock.l_type = F_WRLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
+		    &loc, F_SETLK, &flock, NULL);
 
-                if (pending[j])
-                        return 1;
-        }
+	loc_wipe (&loc);
 
-        return 0;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
 
 int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this)
+afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, off_t off, size_t size,
+		      unsigned char *locked_on)
 {
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
+	afr_local_t *local = NULL;
+	int i = 0;
+	afr_private_t *priv = NULL;
 
-        priv = this->private;
+	priv = this->private;
+	local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-                if (ret != 0)
-                        return 0;
+	flock.l_type = F_WRLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
+		    &loc, F_SETLK, &flock, NULL);
 
-                if (pending[j])
-                        return 1;
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].op_ret == -1 &&
+		    local->replies[i].op_errno == EAGAIN) {
+			afr_selfheal_locked_fill (frame, this, locked_on);
+			afr_selfheal_uninodelk (frame, this, inode, dom, off,
+						size, locked_on);
 
-        return 0;
-}
+			AFR_SEQ (frame, afr_selfheal_lock_cbk, inodelk, dom,
+				 &loc, F_SETLKW, &flock, NULL);
+			break;
+		}
+	}
 
-int
-afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        afr_sh_reset (frame, this);
-        if (local->govinda_gOvinda || sh->op_failed) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "split brain found, aborting selfheal of %s",
-                        local->loc.path);
-                sh->op_failed = 1;
-                sh->completion_cbk (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to metadata check on %s",
-                        local->loc.path);
-                afr_self_heal_metadata (frame, this);
-        }
-
-        return 0;
+	loc_wipe (&loc);
+
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
 
-static int
-afr_sh_missing_entries_finish (call_frame_t *frame, xlator_t *this)
+int
+afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, off_t off, size_t size,
+			const unsigned char *locked_on)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
 
-        local = frame->local;
-        int_lock = &local->internal_lock;
 
-        int_lock->lock_cbk = afr_sh_missing_entries_done;
-        afr_unlock (frame, this);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        return 0;
-}
+	flock.l_type = F_UNLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-int
-afr_sh_common_create (afr_self_heal_t *sh, unsigned int child_count)
-{
-        int     ret = -ENOMEM;
-        sh->buf = GF_CALLOC (child_count, sizeof (*sh->buf),
-                             gf_afr_mt_iatt);
-        if (!sh->buf)
-                goto out;
-        sh->parentbufs = GF_CALLOC (child_count, sizeof (*sh->parentbufs),
-                                    gf_afr_mt_iatt);
-        if (!sh->parentbufs)
-                goto out;
-        sh->child_errno = GF_CALLOC (child_count, sizeof (*sh->child_errno),
-                                     gf_afr_mt_int);
-        if (!sh->child_errno)
-                goto out;
-        sh->success_children = afr_children_create (child_count);
-        if (!sh->success_children)
-                goto out;
-        sh->fresh_children = afr_children_create (child_count);
-        if (!sh->fresh_children)
-                goto out;
-        sh->xattr = GF_CALLOC (child_count, sizeof (*sh->xattr),
-                               gf_afr_mt_dict_t);
-        if (!sh->xattr)
-                goto out;
-        ret = 0;
-out:
-        return ret;
-}
+	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, inodelk,
+		    dom, &loc, F_SETLK, &flock, NULL);
 
-void
-afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   inode_t *inode, struct iatt *buf,
-                                   dict_t *xattr, struct iatt *postparent,
-                                   loc_t *loc)
-{
-        int              child_index = 0;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        priv = this->private;
-        sh   = &local->self_heal;
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        sh->buf[child_index] = *buf;
-                        sh->parentbufs[child_index] = *postparent;
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                        sh->xattr[child_index] = dict_ref (xattr);
-                } else {
-                        gf_log (this->name, GF_LOG_ERROR, "path %s on subvolume"
-                                " %s => -1 (%s)", loc->path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        local->self_heal.child_errno[child_index] = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-        return;
-}
+	loc_wipe (&loc);
 
-gf_boolean_t
-afr_valid_ia_type (ia_type_t ia_type)
-{
-        switch (ia_type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-        case IA_IFDIR:
-                return _gf_true;
-        default:
-                return _gf_false;
-        }
-        return _gf_false;
+	return 0;
 }
 
+
 int
-afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
-                          int active_source, call_frame_t **impunge_frame)
+afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, const char *name, unsigned char *locked_on)
 {
-        afr_local_t     *local         = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh    = NULL;
-        int32_t         op_errno       = 0;
-        afr_private_t   *priv          = NULL;
-        int             ret            = 0;
-        call_frame_t    *new_frame     = NULL;
-
-        op_errno = ENOMEM;
-        priv = this->private;
-        new_frame = copy_frame (frame);
-        if (!new_frame) {
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (impunge_local, out);
-
-        local = frame->local;
-        new_frame->local = impunge_local;
-        impunge_sh = &impunge_local->self_heal;
-        impunge_sh->sh_frame = frame;
-        impunge_sh->active_source = active_source;
-        impunge_local->child_up  = memdup (local->child_up,
-                                           sizeof (*local->child_up) *
-                                           priv->child_count);
-        if (!impunge_local->child_up)
-                goto out;
-
-        impunge_local->pending = afr_matrix_create (priv->child_count,
-                                                    AFR_NUM_CHANGE_LOGS);
-        if (!impunge_local->pending)
-                goto out;
-
-        ret = afr_sh_common_create (impunge_sh, priv->child_count);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        op_errno = 0;
-        *impunge_frame = new_frame;
-out:
-        if (op_errno && new_frame)
-                AFR_STACK_DESTROY (new_frame);
-        return -op_errno;
-}
+	loc_t loc = {0,};
 
-void
-afr_sh_missing_entry_call_impunge_recreate (call_frame_t *frame, xlator_t *this,
-                                            struct iatt *buf,
-                                            struct iatt *postparent,
-                                            afr_impunge_done_cbk_t impunge_done)
-{
-        call_frame_t    *impunge_frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int             ret = 0;
-        unsigned int    enoent_count = 0;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
-        int32_t         op_errno = 0;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        enoent_count = afr_errno_count (NULL, sh->child_errno,
-                                        priv->child_count, ENOENT);
-        if (!enoent_count) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no missing files - %s. proceeding to metadata check",
-                        local->loc.path);
-                goto out;
-        }
-        sh->impunge_done = impunge_done;
-        ret = afr_impunge_frame_create (frame, this, sh->source, &impunge_frame);
-        if (ret)
-                goto out;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-        loc_copy (&impunge_local->loc, &local->loc);
-        ret = afr_build_parent_loc (&impunge_sh->parent_loc,
-                                    &impunge_local->loc, &op_errno);
-        if (ret) {
-                ret = -op_errno;
-                goto out;
-        }
-        impunge_local->call_count = enoent_count;
-        impunge_sh->entrybuf = sh->buf[sh->source];
-        impunge_sh->parentbuf = sh->parentbufs[sh->source];
-        for (i = 0; i < priv->child_count; i++) {
-                if (!impunge_local->child_up[i]) {
-                        impunge_sh->child_errno[i] = ENOTCONN;
-                        continue;
-                }
-                if (sh->child_errno[i] != ENOENT) {
-                        impunge_sh->child_errno[i] = EEXIST;
-                        continue;
-                }
-        }
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->child_errno[i] != ENOENT)
-                        continue;
-                afr_sh_entry_impunge_create (impunge_frame, this, i);
-                enoent_count--;
-        }
-        GF_ASSERT (!enoent_count);
-        return;
-out:
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "impunge of %s failed, "
-                        "reason: %s", local->loc.path, strerror (-ret));
-                sh->op_failed = 1;
-        }
-        afr_sh_missing_entries_finish (frame, this);
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-int
-afr_sh_create_entry_cbk (call_frame_t *frame, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        if (op_ret < 0)
-                sh->op_failed = 1;
-        afr_sh_missing_entries_finish (frame, this);
-        return 0;
-}
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom,
+		   &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-static int
-sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int              type = 0;
-        struct iatt     *buf = NULL;
-        struct iatt     *postparent = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        buf = &sh->buf[sh->source];
-        postparent = &sh->parentbufs[sh->source];
-
-        type = buf->ia_type;
-        if (!afr_valid_ia_type (type)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: unknown file type: 0%o", local->loc.path, type);
-                local->govinda_gOvinda = 1;
-                afr_sh_missing_entries_finish (frame, this);
-                goto out;
-        }
-
-        afr_sh_missing_entry_call_impunge_recreate (frame, this,
-                                                    buf, postparent,
-                                                    afr_sh_create_entry_cbk);
-out:
-        return 0;
-}
+	loc_wipe (&loc);
 
-void
-afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
-                                    int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       ia_type = IA_INVAL;
-        int32_t         nsources = 0;
-        loc_t           *loc = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-        loc = &local->loc;
-
-        if (op_ret < 0) {
-                if (op_errno == EIO)
-                        local->govinda_gOvinda = 1;
-                // EIO can happen if finding the fresh parent dir failed
-                goto out;
-        }
-
-        //now No chance for the ia_type to conflict
-        ia_type = sh->buf[sh->success_children[0]].ia_type;
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      afr_transaction_type_get (ia_type),
-                                      NULL, _gf_false);
-        if (nsources < 0) {
-                gf_log (this->name, GF_LOG_INFO, "No sources for dir of %s,"
-                        " in missing entry self-heal, continuing with the rest"
-                        " of the self-heals", local->loc.path);
-                op_errno = EIO;
-                goto out;
-        }
-
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        sh->source = sh->fresh_children[0];
-        if (sh->source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
-                op_errno = EIO;
-                goto out;
-        }
-
-        if (sh->gfid_sh_success_cbk)
-                sh->gfid_sh_success_cbk (frame, this);
-        sh->type = sh->buf[sh->source].ia_type;
-        if (uuid_is_null (loc->inode->gfid))
-                uuid_copy (loc->gfid, sh->buf[sh->source].ia_gfid);
-        sh_missing_entries_create (frame, this);
-        return;
-out:
-        sh->op_failed = 1;
-        afr_sh_set_error (sh, op_errno);
-        afr_sh_missing_entries_finish (frame, this);
-        return;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
-static int
-afr_sh_common_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *buf, dict_t *xattr,
-                          struct iatt *postparent)
-{
-        int                     call_count = 0;
-        afr_local_t             *local = NULL;
-        afr_self_heal_t         *sh    = NULL;
-        afr_private_t           *priv  = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        afr_sh_common_lookup_resp_handler (frame, cookie, this, op_ret,
-                                           op_errno, inode, buf, xattr,
-                                           postparent, &sh->lookup_loc);
-        call_count = afr_frame_return (frame);
-
-        if (call_count)
-                goto out;
-        op_ret = -1;
-        if (!sh->success_count) {
-                op_errno = afr_resultant_errno_get (NULL, sh->child_errno,
-                                                    priv->child_count);
-                gf_log (this->name, GF_LOG_ERROR, "Failed to lookup %s, "
-                        "reason %s", sh->lookup_loc.path,
-                        strerror (op_errno));
-                goto done;
-        }
-
-        if ((sh->lookup_flags & AFR_LOOKUP_FAIL_CONFLICTS) &&
-            (afr_conflicting_iattrs (sh->buf, sh->success_children,
-                                     priv->child_count,
-                                     sh->lookup_loc.path, this->name))) {
-                op_errno = EIO;
-                gf_log (this->name, GF_LOG_ERROR, "Conflicting entries "
-                        "for %s", sh->lookup_loc.path);
-                goto done;
-        }
-
-        if ((sh->lookup_flags & AFR_LOOKUP_FAIL_MISSING_GFIDS) &&
-            (afr_gfid_missing_count (this->name, sh->success_children,
-                                     sh->buf, priv->child_count,
-                                     sh->lookup_loc.path))) {
-                op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "Missing Gfids "
-                        "for %s", sh->lookup_loc.path);
-                goto done;
-        }
-        op_ret = 0;
-
-done:
-        sh->lookup_done (frame, this, op_ret, op_errno);
-out:
-        return 0;
-}
 
 int
-afr_sh_remove_entry_cbk (call_frame_t *frame, xlator_t *this, int child,
-                         int32_t op_ret, int32_t op_errno)
+afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, const char *name, unsigned char *locked_on)
 {
-        int             call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        GF_ASSERT (sh->post_remove_call);
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "purge entry %s failed, on child %d reason, %s",
-                        local->loc.path, child, strerror (op_errno));
-                LOCK (&frame->lock);
-                {
-                        afr_sh_set_error (sh, EIO);
-                        sh->op_failed = 1;
-                }
-                UNLOCK (&frame->lock);
-        }
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                sh->post_remove_call (frame, this);
-        return 0;
-}
+	loc_t loc = {0,};
+	afr_local_t *local = NULL;
+	int i = 0;
+	afr_private_t *priv = NULL;
 
-void
-afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
-                                  int child_index, struct iatt *buf,
-                                  struct iatt *parentbuf,
-                                  afr_expunge_done_cbk_t expunge_done)
-{
-        call_frame_t    *expunge_frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int32_t         op_errno = 0;
-
-        expunge_frame = copy_frame (frame);
-        if (!expunge_frame) {
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
-
-        local = frame->local;
-        sh = &local->self_heal;
-        expunge_frame->local = expunge_local;
-        expunge_sh = &expunge_local->self_heal;
-        expunge_sh->sh_frame = frame;
-        loc_copy (&expunge_local->loc, &local->loc);
-        sh->expunge_done = expunge_done;
-        afr_sh_entry_expunge_remove (expunge_frame, this, child_index, buf,
-                                     parentbuf);
-        return;
-out:
-        gf_log (this->name, GF_LOG_ERROR, "Expunge of %s failed, reason: %s",
-                local->loc.path, strerror (op_errno));
-        expunge_done (frame, this, child_index, -1, op_errno);
-}
+	priv = this->private;
+	local = frame->local;
 
-void
-afr_sh_remove_stale_lookup_info (afr_self_heal_t *sh, int32_t *success_children,
-                                 int32_t *fresh_children,
-                                 unsigned int child_count)
-{
-        int     i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (afr_is_child_present (success_children, child_count, i) &&
-                    !afr_is_child_present (fresh_children, child_count, i)) {
-                        sh->child_errno[i] = ENOENT;
-                        GF_ASSERT (sh->xattr[i]);
-                        dict_unref (sh->xattr[i]);
-                        sh->xattr[i] = NULL;
-                }
-        }
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-int
-afr_sh_purge_stale_entries_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-        afr_private_t       *priv     = NULL;
-
-        local    = frame->local;
-        sh       = &local->self_heal;
-        priv     = this->private;
-
-        if (sh->op_failed) {
-                afr_sh_missing_entries_finish (frame, this);
-        } else {
-                if (afr_gfid_missing_count (this->name, sh->fresh_children,
-                                            sh->buf, priv->child_count,
-                                            local->loc.path)) {
-                        afr_sh_common_lookup (frame, this, &local->loc,
-                                              afr_sh_missing_entries_lookup_done,
-                                              sh->sh_gfid_req,
-                                              AFR_LOOKUP_FAIL_CONFLICTS|
-                                              AFR_LOOKUP_FAIL_MISSING_GFIDS);
-                } else {
-                        //No need to set gfid so goto missing entries lookup done
-                        //Behave as if you have done the lookup
-                        afr_sh_remove_stale_lookup_info (sh,
-                                                         sh->success_children,
-                                                         sh->fresh_children,
-                                                         priv->child_count);
-                        afr_children_copy (sh->success_children,
-                                           sh->fresh_children,
-                                           priv->child_count);
-                        afr_sh_missing_entries_lookup_done (frame, this, 0, 0);
-                }
-        }
-        return 0;
-}
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
+		   name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-gf_boolean_t
-afr_sh_purge_entry_condition (afr_local_t *local, afr_private_t *priv,
-                              int child)
-{
-        afr_self_heal_t *sh = NULL;
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].op_ret == -1 &&
+		    local->replies[i].op_errno == EAGAIN) {
+			afr_selfheal_locked_fill (frame, this, locked_on);
+			afr_selfheal_unentrylk (frame, this, inode, dom, name,
+						locked_on);
 
-        sh = &local->self_heal;
+			AFR_SEQ (frame, afr_selfheal_lock_cbk, entrylk, dom,
+				 &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+			break;
+		}
+	}
 
-        if (local->child_up[child] &&
-            (!afr_is_child_present (sh->fresh_parent_dirs, priv->child_count,
-                                    child))
-            && (sh->child_errno[child] != ENOENT))
-                return _gf_true;
+	loc_wipe (&loc);
 
-        return _gf_false;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
-gf_boolean_t
-afr_sh_purge_stale_entry_condition (afr_local_t *local, afr_private_t *priv,
-                                    int child)
+
+int
+afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, const char *name, unsigned char *locked_on)
 {
-        afr_self_heal_t *sh = NULL;
+	loc_t loc = {0,};
 
-        sh = &local->self_heal;
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        if (local->child_up[child] &&
-            (!afr_is_child_present (sh->fresh_children, priv->child_count,
-                                    child))
-             && (sh->child_errno[child] != ENOENT))
-                return _gf_true;
+	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, entrylk,
+		    dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
 
-        return _gf_false;
-}
+	loc_wipe (&loc);
 
-void
-afr_sh_purge_entry_common (call_frame_t *frame, xlator_t *this,
-                           gf_boolean_t purge_condition (afr_local_t *local,
-                                                         afr_private_t *priv,
-                                                         int child))
-{
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-        int             i = 0;
-        int             call_count = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (purge_condition (local, priv, i))
-                        call_count++;
-        }
-
-        if (call_count == 0) {
-                sh->post_remove_call (frame, this);
-                goto out;
-        }
-
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!purge_condition (local, priv, i))
-                        continue;
-                gf_log (this->name, GF_LOG_INFO, "purging the stale entry %s "
-                        "on %d", local->loc.path, i);
-                afr_sh_call_entry_expunge_remove (frame, this,
-                                                  (long) i, &sh->buf[i],
-                                                  &sh->parentbufs[i],
-                                                  afr_sh_remove_entry_cbk);
-        }
-out:
-        return;
+	return 0;
 }
 
-void
-afr_sh_purge_entry (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        sh->post_remove_call = afr_sh_missing_entries_finish;
 
-        afr_sh_purge_entry_common (frame, this, afr_sh_purge_entry_condition);
-}
-
-void
-afr_sh_purge_stale_entry (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_pending_set (xlator_t *this, dict_t *xdata, int type)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
+	int idx = -1;
+	afr_private_t *priv = NULL;
+	void *pending_raw = NULL;
+	int *pending_int = NULL;
+	int i = 0;
 
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
+	priv = this->private;
+	idx = afr_index_for_transaction_type (type);
 
-        sh->post_remove_call = afr_sh_purge_stale_entries_done;
+	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw) == 0) {
+		if (pending_raw) {
+			pending_int = pending_raw;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (afr_is_child_present (sh->fresh_children,
-                                          priv->child_count, i))
-                        continue;
+			if (ntoh32 (pending_int[idx]))
+				return _gf_true;
+		}
+	}
 
-                if ((!local->child_up[i]) || sh->child_errno[i] != 0)
-                        continue;
+	for (i = 0; i < priv->child_count; i++) {
+		if (dict_get_ptr (xdata, priv->pending_key[i],
+				  &pending_raw))
+			continue;
+		if (!pending_raw)
+			continue;
+		pending_int = pending_raw;
 
-                GF_ASSERT (!uuid_is_null (sh->entrybuf.ia_gfid) ||
-                           uuid_is_null (sh->buf[i].ia_gfid));
+		if (ntoh32 (pending_int[idx]))
+			return _gf_true;
+	}
 
-                if ((sh->entrybuf.ia_type != sh->buf[i].ia_type) ||
-                    (uuid_compare (sh->buf[i].ia_gfid,
-                                   sh->entrybuf.ia_gfid)))
-                        continue;
+	return _gf_false;
+}
 
-                afr_children_add_child (sh->fresh_children, i,
-                                        priv->child_count);
 
-        }
-        afr_sh_purge_entry_common (frame, this,
-                                   afr_sh_purge_stale_entry_condition);
+gf_boolean_t
+afr_is_data_set (xlator_t *this, dict_t *xdata)
+{
+	return afr_is_pending_set (this, xdata, AFR_DATA_TRANSACTION);
 }
 
-void
-afr_sh_save_child_iatts_from_policy (int32_t *children, struct iatt *bufs,
-                                     struct iatt *save,
-                                     unsigned int child_count)
+gf_boolean_t
+afr_is_metadata_set (xlator_t *this, dict_t *xdata)
 {
-        int             i = 0;
-        int             child = 0;
-        gf_boolean_t    saved = _gf_false;
-
-        GF_ASSERT (save);
-        //if iatt buf with gfid exists sets it
-        for (i = 0; i < child_count; i++) {
-                child = children[i];
-                if (child == -1)
-                        break;
-                *save = bufs[child];
-                saved = _gf_true;
-                if (!uuid_is_null (save->ia_gfid))
-                        break;
-        }
-        GF_ASSERT (saved);
+	return afr_is_pending_set (this, xdata, AFR_METADATA_TRANSACTION);
 }
 
-void
-afr_get_children_of_fresh_parent_dirs (afr_self_heal_t *sh,
-                                       unsigned int child_count)
+gf_boolean_t
+afr_is_entry_set (xlator_t *this, dict_t *xdata)
 {
-        afr_children_intersection_get (sh->success_children,
-                                       sh->fresh_parent_dirs,
-                                       sh->sources, child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, child_count);
-        memset (sh->sources, 0, sizeof (*sh->sources) * child_count);
+	return afr_is_pending_set (this, xdata, AFR_ENTRY_TRANSACTION);
 }
 
+
 void
-afr_sh_children_lookup_done (call_frame_t *frame, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+afr_inode_link (inode_t *inode, struct iatt *iatt)
 {
-        afr_local_t      *local = NULL;
-        afr_self_heal_t  *sh = NULL;
-        afr_private_t    *priv = NULL;
-        int32_t          fresh_child_enoents = 0;
-        int32_t          fresh_parent_count = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto fail;
-        afr_get_children_of_fresh_parent_dirs (sh, priv->child_count);
-        fresh_parent_count = afr_get_children_count (sh->fresh_parent_dirs,
-                                                     priv->child_count);
-        //we need the enoent count of the subvols present in fresh_parent_dirs
-        fresh_child_enoents = afr_errno_count (sh->fresh_parent_dirs,
-                                               sh->child_errno,
-                                               priv->child_count, ENOENT);
-        if (fresh_child_enoents == fresh_parent_count) {
-                gf_log (this->name, GF_LOG_INFO, "Deleting stale file %s",
-                        local->loc.path);
-                afr_sh_set_error (sh, ENOENT);
-                sh->op_failed = 1;
-                afr_sh_purge_entry (frame, this);
-        } else if (!afr_conflicting_iattrs (sh->buf, sh->fresh_children,
-                                            priv->child_count, local->loc.path,
-                                            this->name)) {
-                afr_sh_save_child_iatts_from_policy (sh->fresh_children,
-                                                     sh->buf, &sh->entrybuf,
-                                                     priv->child_count);
-                afr_update_gfid_from_iatts (sh->sh_gfid_req, sh->buf,
-                                            sh->fresh_children,
-                                            priv->child_count);
-                afr_sh_purge_stale_entry (frame, this);
-        } else {
-                op_errno = EIO;
-                local->govinda_gOvinda = 1;
-                goto fail;
-        }
-
-        return;
-
-fail:
-        sh->op_failed = 1;
-        afr_sh_set_error (sh, op_errno);
-        afr_sh_missing_entries_finish (frame, this);
-        return;
-}
+	inode_t *linked_inode = NULL;
 
-static void
-afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
-{
-        afr_self_heal_t *sh  = NULL;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local = NULL;
-        int             enoent_count = 0;
-        int             nsources = 0;
-        int             source  = -1;
-        int32_t         subvol_status = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto out;
-        enoent_count = afr_errno_count (NULL, sh->child_errno,
-                                        priv->child_count, ENOENT);
-        if (enoent_count > 0) {
-                gf_log (this->name, GF_LOG_INFO, "Parent dir missing for %s,"
-                        " in missing entry self-heal, aborting missing-entry "
-                        "self-heal",
-                        local->loc.path);
-                afr_sh_missing_entries_finish (frame, this);
-                return;
-        }
-
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_ENTRY_TRANSACTION, &subvol_status,
-                                      _gf_true);
-        if ((subvol_status & ALL_FOOLS) ||
-            (subvol_status & SPLIT_BRAIN)) {
-                gf_log (this->name, GF_LOG_INFO, "%s: Performing conservative "
-                        "merge", sh->parent_loc.path);
-                afr_mark_success_children_sources (sh->sources,
-                                                   sh->success_children,
-                                                   priv->child_count);
-        } else if (nsources < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "No sources for dir "
-                        "of %s, in missing entry self-heal, aborting "
-                        "self-heal", local->loc.path);
-                goto out;
-        }
-
-        source = afr_sh_select_source (sh->sources, priv->child_count);
-        if (source == -1) {
-                GF_ASSERT (0);
-                gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
-                goto out;
-        }
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_parent_dirs, priv->child_count);
-        afr_sh_common_lookup (frame, this, &local->loc,
-                              afr_sh_children_lookup_done, NULL, 0);
-        return;
+	linked_inode = inode_link (inode, NULL, NULL, iatt);
 
-out:
-        afr_sh_set_error (sh, EIO);
-        sh->op_failed = 1;
-        afr_sh_missing_entries_finish (frame, this);
-        return;
-}
+	uuid_copy (inode->gfid, iatt->ia_gfid);
+	inode->ia_type = iatt->ia_type;
 
-void
-afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count)
-{
-        int             i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                memset (&sh->buf[i], 0, sizeof (sh->buf[i]));
-                memset (&sh->parentbufs[i], 0, sizeof (sh->parentbufs[i]));
-                sh->child_errno[i] = 0;
-        }
-        memset (&sh->parentbuf, 0, sizeof (sh->parentbuf));
-        sh->success_count = 0;
-        afr_reset_children (sh->success_children, child_count);
-        afr_reset_children (sh->fresh_children, child_count);
-        afr_reset_xattr (sh->xattr, child_count);
-        loc_wipe (&sh->lookup_loc);
+	if (linked_inode) {
+		inode_lookup (linked_inode);
+		inode_unref (linked_inode);
+	}
 }
 
-/* afr self-heal state will be lost if this call is made
- * please check the afr_sh_common_reset that is called in this function
+
+/*
+ * This function inspects the looked up replies (in an unlocked manner)
+ * and decides whether a locked verification and possible healing is
+ * required or not. It updates the three booleans for each type
+ * of healing. If the boolean flag gets set to FALSE, then we are sure
+ * no healing is required. If the boolean flag gets set to TRUE then
+ * we have to proceed with locked reinspection.
  */
+
 int
-afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      afr_lookup_done_cbk_t lookup_done , uuid_t gfid,
-                      int32_t flags)
-{
-        afr_local_t    *local = NULL;
-        int             i = 0;
-        int             call_count = 0;
-        afr_private_t  *priv = NULL;
-        dict_t         *xattr_req = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-        sh    = &local->self_heal;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        local->call_count = call_count;
-
-        xattr_req = dict_new();
-
-        if (xattr_req) {
-                afr_xattr_req_prepare (this, xattr_req, loc->path);
-                if (gfid) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "looking up %s with gfid: %s",
-                                loc->path, uuid_utoa (gfid));
-                        GF_ASSERT (!uuid_is_null (gfid));
-                        afr_set_dict_gfid (xattr_req, gfid);
-                }
-        }
-
-        afr_sh_common_reset (sh, priv->child_count);
-        sh->lookup_done = lookup_done;
-        loc_copy (&sh->lookup_loc, loc);
-        sh->lookup_flags = flags;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "looking up %s on subvolume %s",
-                                loc->path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame,
-                                           afr_sh_common_lookup_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           loc, xattr_req);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        return 0;
-}
+afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
+			       inode_t *inode, uuid_t gfid,
+			       gf_boolean_t *data_selfheal,
+			       gf_boolean_t *metadata_selfheal,
+			       gf_boolean_t *entry_selfheal)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int valid_cnt = 0;
+	struct iatt first = {0, };
+	struct afr_reply *replies = NULL;
+	int ret = -1;
+
+	priv = this->private;
+
+	replies = alloca0 (sizeof (*replies) * priv->child_count);
+
+	ret = afr_selfheal_unlocked_discover (frame, inode, gfid, replies);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+		if (replies[i].op_ret == -1)
+			continue;
+
+		if (afr_is_data_set (this, replies[i].xdata))
+			*data_selfheal = _gf_true;
+
+		if (afr_is_metadata_set (this, replies[i].xdata))
+			*metadata_selfheal = _gf_true;
 
+		if (afr_is_entry_set (this, replies[i].xdata))
+			*entry_selfheal = _gf_true;
 
+		valid_cnt ++;
+		if (valid_cnt == 1) {
+			first = replies[i].poststat;
+			continue;
+		}
 
-int
-afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Non blocking entrylks failed.");
-                sh->op_failed = -1;
-                afr_sh_missing_entries_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_sh_common_lookup (frame, this, &sh->parent_loc,
-                                      afr_sh_find_fresh_parents,
-                                      NULL, AFR_LOOKUP_FAIL_CONFLICTS);
-        }
-
-        return 0;
-}
+		if (!IA_EQUAL (first, replies[i].poststat, type)) {
+			gf_log (this->name, GF_LOG_ERROR,
+				"TYPE mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_type,
+				(int) replies[i].poststat.ia_type,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
+			return -EIO;
+		}
 
-int
-afr_sh_post_nb_entrylk_gfid_sh_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        sh       = &local->self_heal;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Non blocking entrylks failed.");
-                afr_sh_missing_entries_done (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_missing_entries_lookup_done,
-                                      sh->sh_gfid_req, AFR_LOOKUP_FAIL_CONFLICTS|
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
-        }
-
-        return 0;
-}
+		if (!IA_EQUAL (first, replies[i].poststat, uid)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"UID mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_uid,
+				(int) replies[i].poststat.ia_uid,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-int
-afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                char *base_name, afr_lock_cbk_t lock_cbk)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+			*metadata_selfheal = _gf_true;
+		}
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+		if (!IA_EQUAL (first, replies[i].poststat, gid)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"GID mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_uid,
+				(int) replies[i].poststat.ia_uid,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_ENTRY_SELF_HEAL_LK;
+			*metadata_selfheal = _gf_true;
+		}
 
-        afr_set_lock_number (frame, this);
+		if (!IA_EQUAL (first, replies[i].poststat, prot)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"MODE mismatch %d vs %d on %s for gfid:%s",
+				(int) st_mode_from_ia (first.ia_prot, 0),
+				(int) st_mode_from_ia (replies[i].poststat.ia_prot, 0),
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-        int_lock->lk_basename = base_name;
-        int_lock->lk_loc      = loc;
-        int_lock->lock_cbk    = lock_cbk;
+			*metadata_selfheal = _gf_true;
+		}
 
-        afr_nonblocking_entrylk (frame, this);
+		if (IA_ISREG(first.ia_type) &&
+		    !IA_EQUAL (first, replies[i].poststat, size)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"SIZE mismatch %lld vs %lld on %s for gfid:%s",
+				(long long) first.ia_size,
+				(long long) replies[i].poststat.ia_size,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-        return 0;
-}
+			*data_selfheal = _gf_true;
+		}
+	}
 
-static int
-afr_self_heal_parent_entrylk (call_frame_t *frame, xlator_t *this,
-                              afr_lock_cbk_t lock_cbk)
-{
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int                 ret       = -1;
-        int32_t             op_errno  = 0;
-
-        local    = frame->local;
-        sh       = &local->self_heal;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "attempting to recreate missing entries for path=%s",
-                local->loc.path);
-
-        ret = afr_build_parent_loc (&sh->parent_loc, &local->loc, &op_errno);
-        if (ret)
-                goto out;
-
-        afr_sh_entrylk (frame, this, &sh->parent_loc, NULL,
-                        lock_cbk);
-        return 0;
-out:
-        int_lock = &local->internal_lock;
-        int_lock->lock_op_ret = -1;
-        lock_cbk (frame, this);
-        return 0;
-}
+	if (valid_cnt > 0)
+		afr_inode_link (inode, &first);
 
-static int
-afr_self_heal_conflicting_entries (call_frame_t *frame, xlator_t *this)
-{
-        afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_conflicting_sh_cbk);
-        return 0;
-}
+	if (valid_cnt < 2)
+		return -ENOTCONN;
 
-static int
-afr_self_heal_gfids (call_frame_t *frame, xlator_t *this)
-{
-        afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_gfid_sh_cbk);
-        return 0;
+	return 0;
 }
 
-afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
+
+inode_t *
+afr_inode_find (xlator_t *this, uuid_t gfid)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *lc     = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *shc = NULL;
-
-        priv = this->private;
-
-        sh = &l->self_heal;
-
-        lc = mem_get0 (this->local_pool);
-        if (!lc)
-                goto out;
-
-        shc = &lc->self_heal;
-
-        shc->unwind = sh->unwind;
-        shc->gfid_sh_success_cbk = sh->gfid_sh_success_cbk;
-        shc->do_missing_entry_self_heal = sh->do_missing_entry_self_heal;
-        shc->do_gfid_self_heal = sh->do_gfid_self_heal;
-        shc->do_data_self_heal = sh->do_data_self_heal;
-        shc->do_metadata_self_heal = sh->do_metadata_self_heal;
-        shc->do_entry_self_heal = sh->do_entry_self_heal;
-        shc->forced_merge = sh->forced_merge;
-        shc->background = sh->background;
-        shc->type = sh->type;
-
-        uuid_copy (shc->sh_gfid_req, sh->sh_gfid_req);
-        if (l->loc.path)
-                loc_copy (&lc->loc, &l->loc);
-
-        lc->child_up  = memdup (l->child_up,
-                                sizeof (*lc->child_up) * priv->child_count);
-        if (l->xattr_req)
-                lc->xattr_req = dict_ref (l->xattr_req);
-
-        if (l->cont.lookup.inode)
-                lc->cont.lookup.inode = inode_ref (l->cont.lookup.inode);
-        if (l->cont.lookup.xattr)
-                lc->cont.lookup.xattr = dict_ref (l->cont.lookup.xattr);
-        if (l->internal_lock.inode_locked_nodes)
-                lc->internal_lock.inode_locked_nodes =
-                        memdup (l->internal_lock.inode_locked_nodes,
-                                sizeof (*lc->internal_lock.inode_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.inode_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.inode_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.entry_locked_nodes)
-                lc->internal_lock.entry_locked_nodes =
-                        memdup (l->internal_lock.entry_locked_nodes,
-                                sizeof (*lc->internal_lock.entry_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.entry_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.entry_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.locked_nodes)
-                lc->internal_lock.locked_nodes =
-                        memdup (l->internal_lock.locked_nodes,
-                                sizeof (*lc->internal_lock.locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-
-        lc->internal_lock.inodelk_lock_count =
-                l->internal_lock.inodelk_lock_count;
-        lc->internal_lock.entrylk_lock_count =
-                l->internal_lock.entrylk_lock_count;
+	inode_table_t *table = NULL;
+	inode_t *inode = NULL;
 
-out:
-        return lc;
-}
+	table = this->itable;
+	if (!table)
+		return NULL;
 
-int
-afr_self_heal_completion_cbk (call_frame_t *bgsh_frame, xlator_t *this)
-{
-        afr_private_t *   priv  = NULL;
-        afr_local_t *     local = NULL;
-        afr_self_heal_t * sh    = NULL;
-        char              sh_type_str[256] = {0,};
-        gf_boolean_t      split_brain = _gf_false;
-
-        priv  = this->private;
-        local = bgsh_frame->local;
-        sh    = &local->self_heal;
-
-        if (local->govinda_gOvinda)
-                split_brain = _gf_true;
-
-        afr_set_split_brain (this, sh->inode, split_brain);
-
-        afr_self_heal_type_str_get (sh, sh_type_str,
-                                    sizeof(sh_type_str));
-        if (sh->op_failed) {
-                gf_log (this->name, GF_LOG_ERROR, "background %s self-heal "
-                        "failed on %s", sh_type_str, local->loc.path);
-        } else {
-                gf_log (this->name, GF_LOG_INFO, "background %s self-heal "
-                        "completed on %s", sh_type_str, local->loc.path);
-        }
-
-        FRAME_SU_UNDO (bgsh_frame, afr_local_t);
-
-        if (!sh->unwound && sh->unwind) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno);
-        }
-
-        if (sh->background) {
-                LOCK (&priv->lock);
-                {
-                        priv->background_self_heals_started--;
-                }
-                UNLOCK (&priv->lock);
-        }
-
-        AFR_STACK_DESTROY (bgsh_frame);
-
-        return 0;
-}
+	inode = inode_find (table, gfid);
+	if (inode)
+		return inode;
 
-int
-afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int32_t          op_errno = 0;
-        int              ret = 0;
-        afr_self_heal_t *orig_sh = NULL;
-        call_frame_t    *sh_frame = NULL;
-        afr_local_t     *sh_local = NULL;
-        loc_t           *loc   = NULL;
-
-        local = frame->local;
-        orig_sh = &local->self_heal;
-        priv  = this->private;
-
-        GF_ASSERT (local->loc.path);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "performing self heal on %s (metadata=%d data=%d entry=%d)",
-                local->loc.path,
-                local->self_heal.do_metadata_self_heal,
-                local->self_heal.do_data_self_heal,
-                local->self_heal.do_entry_self_heal);
-
-        op_errno        = ENOMEM;
-        sh_frame        = copy_frame (frame);
-        if (!sh_frame)
-                goto out;
-        afr_set_lk_owner (sh_frame, this);
-        afr_set_low_priority (sh_frame);
-
-        sh_local        = afr_local_copy (local, this);
-        if (!sh_local)
-                goto out;
-        sh_frame->local = sh_local;
-        sh              = &sh_local->self_heal;
-
-        sh->inode       = inode_ref (inode);
-
-        sh->orig_frame  = frame;
-
-        sh->completion_cbk = afr_self_heal_completion_cbk;
-
-        sh->success = GF_CALLOC (priv->child_count, sizeof (*sh->success),
-                                 gf_afr_mt_char);
-        if (!sh->success)
-                goto out;
-        sh->sources = GF_CALLOC (sizeof (*sh->sources), priv->child_count,
-                                 gf_afr_mt_int);
-        if (!sh->sources)
-                goto out;
-        sh->locked_nodes = GF_CALLOC (sizeof (*sh->locked_nodes),
-                                      priv->child_count,
-                                      gf_afr_mt_int);
-        if (!sh->locked_nodes)
-                goto out;
-
-        sh->pending_matrix = afr_matrix_create (priv->child_count,
-                                                priv->child_count);
-        if (!sh->pending_matrix)
-                goto out;
-
-        sh->delta_matrix = afr_matrix_create (priv->child_count,
-                                              priv->child_count);
-        if (!sh->delta_matrix)
-                goto out;
-
-        sh->fresh_parent_dirs = afr_children_create (priv->child_count);
-        if (!sh->fresh_parent_dirs)
-                goto out;
-        ret = afr_sh_common_create (sh, priv->child_count);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        if (local->self_heal.background) {
-                LOCK (&priv->lock);
-                {
-                        if (priv->background_self_heals_started
-                            < priv->background_self_heal_count) {
-                                priv->background_self_heals_started++;
-
-
-                        } else {
-                                local->self_heal.background = _gf_false;
-                                sh->background = _gf_false;
-                        }
-                }
-                UNLOCK (&priv->lock);
-        }
-
-        if (!local->loc.parent) {
-                sh->do_missing_entry_self_heal = _gf_false;
-                sh->do_gfid_self_heal = _gf_false;
-        }
-
-        FRAME_SU_DO (sh_frame, afr_local_t);
-        if (sh->do_missing_entry_self_heal) {
-                afr_self_heal_conflicting_entries (sh_frame, this);
-        } else if (sh->do_gfid_self_heal) {
-                GF_ASSERT (!uuid_is_null (sh->sh_gfid_req));
-                afr_self_heal_gfids (sh_frame, this);
-        } else {
-                loc = &sh_local->loc;
-                if (uuid_is_null (loc->inode->gfid) && uuid_is_null (loc->gfid)) {
-                        if (!uuid_is_null (inode->gfid))
-                                GF_ASSERT (!uuid_compare (inode->gfid,
-                                           sh->sh_gfid_req));
-                        uuid_copy (loc->gfid, sh->sh_gfid_req);
-                }
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to metadata check on %s",
-                        local->loc.path);
-
-                afr_sh_missing_entries_done (sh_frame, this);
-        }
-        op_errno = 0;
+	inode = inode_new (table);
+	if (!inode)
+		return NULL;
 
-out:
-        if (op_errno) {
-                orig_sh->unwind (frame, this, -1, op_errno);
-                if (sh_frame)
-                        AFR_STACK_DESTROY (sh_frame);
-        }
-        return 0;
+	uuid_copy (inode->gfid, gfid);
+
+	return inode;
 }
 
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size)
+
+call_frame_t *
+afr_frame_create (xlator_t *this)
 {
-        GF_ASSERT (str && (size > strlen (" missing-entry gfid "
-                                          "meta-data data entry")));
+	call_frame_t *frame = NULL;
+	afr_local_t *local = NULL;
+	int op_errno = 0;
+	pid_t pid = -1;
 
-        if (self_heal_p->do_metadata_self_heal) {
-                snprintf (str, size, " meta-data");
-        }
+	frame = create_frame (this, this->ctx->pool);
+	if (!frame)
+		return NULL;
 
-        if (self_heal_p->do_data_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " data");
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local) {
+		STACK_DESTROY (frame->root);
+		return NULL;
+	}
 
-        if (self_heal_p->do_entry_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " entry");
-        }
+	syncopctx_setfspid (&pid);
 
-        if (self_heal_p->do_missing_entry_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str),
-                         " missing-entry");
-        }
+	frame->root->pid = pid;
 
-        if (self_heal_p->do_gfid_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " gfid");
-        }
-}
+	afr_set_lk_owner (frame, this, frame->root);
 
-afr_self_heal_type
-afr_self_heal_type_for_transaction (afr_transaction_type type)
-{
-        afr_self_heal_type sh_type = AFR_SELF_HEAL_INVALID;
-
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_DATA;
-                break;
-        case AFR_METADATA_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_METADATA;
-                break;
-        case AFR_ENTRY_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_ENTRY;
-                break;
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                GF_ASSERT (0);
-                break;
-        }
-        return sh_type;
+	return frame;
 }
 
-int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
-{
-        int   ret = -1;
-        uuid_t pargfid = {0};
 
-        if (!child)
-                goto out;
+/*
+ * This is the entry point for healing a given GFID
+ */
 
-        if (!uuid_is_null (parent->inode->gfid))
-                uuid_copy (pargfid, parent->inode->gfid);
-        else if (!uuid_is_null (parent->gfid))
-                uuid_copy (pargfid, parent->gfid);
+int
+afr_selfheal (xlator_t *this, uuid_t gfid)
+{
+	inode_t *inode = NULL;
+	call_frame_t *frame = NULL;
+	int ret = -1;
+	gf_boolean_t data_selfheal = _gf_false;
+	gf_boolean_t metadata_selfheal = _gf_false;
+	gf_boolean_t entry_selfheal = _gf_false;
 
-        if (uuid_is_null (pargfid))
-                goto out;
+	inode = afr_inode_find (this, gfid);
+	if (!inode)
+		goto out;
 
-        if (strcmp (parent->path, "/") == 0)
-                ret = gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
-                                   name);
+	frame = afr_frame_create (this);
+	if (!frame)
+		goto out;
 
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "asprintf failed while setting child path");
-        }
+	ret = afr_selfheal_unlocked_inspect (frame, this, inode, gfid,
+					     &data_selfheal,
+					     &metadata_selfheal,
+					     &entry_selfheal);
+	if (ret)
+		goto out;
 
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
+	if (data_selfheal)
+		afr_selfheal_data (frame, this, inode);
 
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
-        uuid_copy (child->pargfid, pargfid);
+	if (metadata_selfheal)
+		afr_selfheal_metadata (frame, this, inode);
 
-        if (!child->inode) {
-                ret = -1;
-                goto out;
-        }
+	if (entry_selfheal)
+		afr_selfheal_entry (frame, this, inode);
 
-        ret = 0;
+	inode_forget (inode, 1);
 out:
-        if ((ret == -1) && child)
-                loc_wipe (child);
+	if (inode)
+		inode_unref (inode);
+	if (frame)
+		AFR_STACK_DESTROY (frame);
 
-        return ret;
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.h b/xlators/cluster/afr/src/afr-self-heal-common.h
deleted file mode 100644
index 24cac6232..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-common.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __AFR_SELF_HEAL_COMMON_H__
-#define __AFR_SELF_HEAL_COMMON_H__
-
-#define FILE_HAS_HOLES(buf) (((buf)->ia_size) > ((buf)->ia_blocks * 512))
-
-typedef enum {
-        AFR_SELF_HEAL_ENTRY,
-        AFR_SELF_HEAL_METADATA,
-        AFR_SELF_HEAL_DATA,
-        AFR_SELF_HEAL_INVALID = -1,
-} afr_self_heal_type;
-
-typedef enum {
-        AFR_LOOKUP_FAIL_CONFLICTS = 1,
-        AFR_LOOKUP_FAIL_MISSING_GFIDS = 2,
-} afr_lookup_flags_t;
-
-int
-afr_sh_select_source (int sources[], int child_count);
-
-int
-afr_sh_source_count (int sources[], int child_count);
-
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
-
-int
-afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
-                          unsigned char *ignorant_subvols,
-                          dict_t *xattr[], afr_transaction_type type,
-                          size_t child_count);
-
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], unsigned char success[],
-                         int child_count, afr_transaction_type type);
-
-int
-afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
-                  struct iatt *bufs, afr_self_heal_type type,
-                  int32_t *success_children, int32_t *subvol_status);
-
-int
-afr_sh_delta_to_xattr (afr_private_t *priv,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, afr_transaction_type type);
-
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size);
-
-afr_self_heal_type
-afr_self_heal_type_for_transaction (afr_transaction_type type);
-
-int
-afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
-                   int32_t **pending_matrix, int32_t *sources,
-                   int32_t *success_children, afr_transaction_type type,
-                   int32_t *subvol_status, gf_boolean_t ignore_ignorant);
-void
-afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count);
-
-void
-afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   inode_t *inode, struct iatt *buf,
-                                   dict_t *xattr, struct iatt *postparent,
-                                   loc_t *loc);
-
-int
-afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      afr_lookup_done_cbk_t lookup_cbk, uuid_t uuid,
-                      int32_t flags);
-int
-afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src, struct iatt *buf,
-                             struct iatt *parentbuf);
-int
-afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                char *base_name, afr_lock_cbk_t lock_cbk);
-int
-afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
-                             int child_index);
-int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
-                    afr_lock_cbk_t lock_cbk);
-afr_local_t *
-afr_local_copy (afr_local_t *l, xlator_t *this);
-int
-afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
-                  afr_lock_cbk_t success_handler,
-                  afr_lock_cbk_t failure_handler);
-void
-afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno);
-void
-afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this);
-typedef int
-(*afr_fxattrop_cbk_t) (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr);
-int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
-int
-afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
-                          int active_source, call_frame_t **impunge_frame);
-void
-afr_sh_reset (call_frame_t *frame, xlator_t *this);
-
-void
-afr_children_intersection_get (int32_t *set1, int32_t *set2,
-                               int *intersection, unsigned int child_count);
-#endif /* __AFR_SELF_HEAL_COMMON_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 765edd277..c0548d995 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1,1332 +1,635 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
-
-
-extern int
-sh_loop_finish (call_frame_t *loop_frame, xlator_t *this);
-
-int
-afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this);
-
-int
-afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
-                      afr_fxattrop_cbk_t fxattrop_cbk);
-
-int
-afr_post_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xattr);
-
-int
-afr_sh_data_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->completion_cbk (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t   *local       = NULL;
-        afr_private_t *priv        = NULL;
-        int            call_count  = 0;
-        int            child_index = (long) cookie;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "flush failed on %s on subvolume %s: %s",
-                                local->loc.path, priv->children[child_index]->name,
-                                strerror (op_errno));
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_sh_data_done (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_data_close (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-        int              i          = 0;
-        int              call_count = 0;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        call_count        = afr_set_elem_count_get (sh->success,
-                                                    priv->child_count);
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                afr_sh_data_done (frame, this);
-                return 0;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sh->success[i])
-                        continue;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "closing fd of %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->flush,
-                                   sh->healing_fd);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
-int
-afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                         struct iatt *statpost)
-{
-
-        afr_local_t   *local       = NULL;
-        afr_private_t *priv        = NULL;
-        int            call_count  = 0;
-        int            child_index = (long) cookie;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "setattr failed on %s on subvolume %s: %s",
-                                local->loc.path, priv->children[child_index]->name,
-                                strerror (op_errno));
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_sh_data_finish (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_data_setattr (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-        int              i          = 0;
-        int              call_count = 0;
-        int              source     = 0;
-        int32_t          valid      = 0;
-        struct iatt      stbuf      = {0,};
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        source = sh->source;
-
-        valid |= (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-        stbuf.ia_atime = sh->buf[source].ia_atime;
-        stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-        stbuf.ia_mtime = sh->buf[source].ia_mtime;
-        stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
-
-        call_count        = afr_set_elem_count_get (sh->success,
-                                                    priv->child_count);
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                GF_ASSERT (0);
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sh->success[i])
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_setattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid);
+#include "byte-order.h"
 
-                if (!--call_count)
-                        break;
-        }
+enum {
+	AFR_SELFHEAL_DATA_FULL = 0,
+	AFR_SELFHEAL_DATA_DIFF,
+};
 
-        return 0;
-}
 
-int
-afr_sh_data_setattr_fstat_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               struct iatt *buf)
+#define HAS_HOLES(i) ((i->ia_blocks * 512) < (i->ia_size))
+static int
+__checksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, uint32_t weak, uint8_t *strong,
+		dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int child_index = (long) cookie;
+	afr_local_t *local = NULL;
+	int i = (long) cookie;
 
-        local = frame->local;
-        sh = &local->self_heal;
+	local = frame->local;
 
-        GF_ASSERT (sh->source == child_index);
-        if (op_ret != -1)
-                sh->buf[child_index] = *buf;
-        afr_sh_data_setattr (frame, this);
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (strong)
+		memcpy (local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
 
-        return 0;
+	syncbarrier_wake (&local->barrier);
+	return 0;
 }
 
-/*
- * If there are any writes after the self-heal is triggered then the
- * stbuf stored in local->self_heal.buf[] will be invalid so we do one more
- * stat on the source and then set the [am]times
- */
-int
-afr_sh_set_timestamps (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        STACK_WIND_COOKIE (frame, afr_sh_data_setattr_fstat_cbk,
-                           (void *) (long) sh->source,
-                           priv->children[sh->source],
-                           priv->children[sh->source]->fops->fstat,
-                           sh->healing_fd);
-        return 0;
-}
-
-//Fun fact, lock_cbk is being used for both lock & unlock
-int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
-                    afr_lock_cbk_t lock_cbk)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        GF_ASSERT (sh->data_lock_held);
-
-        sh->data_lock_held = _gf_false;
-        int_lock->lock_cbk = lock_cbk;
-        afr_unlock (frame, this);
-
-        return 0;
-}
 
-int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
+static int
+attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	  int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+	  dict_t *xdata)
 {
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "finishing data selfheal of %s", local->loc.path);
+	int i = (long) cookie;
+	afr_local_t *local = NULL;
 
-        if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
-        else
-                afr_sh_data_close (frame, this);
+	local = frame->local;
 
-        return 0;
-}
-
-int
-afr_sh_data_fail (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (pre)
+		local->replies[i].prestat = *pre;
+	if (post)
+		local->replies[i].poststat = *post;
+	if (xdata)
+		local->replies[i].xdata = dict_ref (xdata);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "finishing failed data selfheal of %s", local->loc.path);
+	syncbarrier_wake (&local->barrier);
 
-        sh->op_failed = 1;
-        if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
-        else
-                afr_sh_data_close (frame, this);
-        return 0;
+	return 0;
 }
 
-int
-afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, dict_t *xattr)
-{
-        int             call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local = frame->local;
-                sh = &local->self_heal;
-                if (!IA_ISREG (sh->type)) {
-                        afr_sh_data_finish (frame, this);
-                        goto out;
-                }
-                GF_ASSERT (sh->old_loop_frame);
-                afr_sh_data_lock (frame, this, 0, 0,
-                                  afr_post_sh_big_lock_success,
-                                  afr_post_sh_big_lock_failure);
-        }
-out:
-        return 0;
-}
 
-int
-afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
+static gf_boolean_t
+__afr_selfheal_data_checksums_match (call_frame_t *frame, xlator_t *this,
+				     fd_t *fd, int source,
+				     unsigned char *healed_sinks,
+				     off_t offset, size_t size)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_DATA_TRANSACTION);
-        gf_log (this->name, GF_LOG_DEBUG, "Delta matrix for: %s",
-                lkowner_utoa (&frame->root->lk_owner));
-        afr_sh_print_pending_matrix (sh->delta_matrix, this);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_DATA_TRANSACTION);
-
-        GF_ASSERT (call_count);
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->fxattrop,
-                                   sh->healing_fd,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	unsigned char *wind_subvols = NULL;
+	int i = 0;
 
-        return 0;
-}
+	priv = this->private;
+	local = frame->local;
 
+	wind_subvols = alloca0 (priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (i == source || healed_sinks[i])
+			wind_subvols[i] = 1;
+	}
 
-static struct afr_sh_algorithm *
-sh_algo_from_name (xlator_t *this, char *name)
-{
-        int i = 0;
+	AFR_ONLIST (wind_subvols, frame, __checksum_cbk, rchecksum, fd,
+		    offset, size, NULL);
 
-        while (afr_self_heal_algorithms[i].name) {
-                if (!strcmp (name, afr_self_heal_algorithms[i].name)) {
-                        return &afr_self_heal_algorithms[i];
-                }
+	if (!local->replies[source].valid || local->replies[source].op_ret != 0)
+		return _gf_false;
 
-                i++;
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (i == source)
+			continue;
+		if (memcmp (local->replies[source].checksum,
+			    local->replies[i].checksum,
+			    MD5_DIGEST_LENGTH))
+			return _gf_false;
+	}
 
-        return NULL;
+	return _gf_true;
 }
 
 
 static int
-sh_zero_byte_files_exist (afr_self_heal_t *sh, int child_count)
-{
-        int i;
-        int ret = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (sh->buf[i].ia_size == 0) {
-                        ret = 1;
-                        break;
-                }
-        }
-
-        return ret;
-}
-
-
-struct afr_sh_algorithm *
-afr_sh_data_pick_algo (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *           priv  = NULL;
-        struct afr_sh_algorithm * algo  = NULL;
-        afr_local_t *             local = NULL;
-        afr_self_heal_t *         sh    = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        algo  = sh_algo_from_name (this, priv->data_self_heal_algorithm);
-
-        if (algo == NULL) {
-                /* option not set, so fall back on heuristics */
-
-                if ((local->enoent_count != 0)
-                    || sh_zero_byte_files_exist (sh, priv->child_count)
-                    || (sh->file_size <= (priv->data_self_heal_window_size *
-                                          this->ctx->page_size))) {
-
-                        /*
-                         * If the file does not exist on one of the subvolumes,
-                         * or a zero-byte file exists (created by entry self-heal)
-                         * the entire content has to be copied anyway, so there
-                         * is no benefit from using the "diff" algorithm.
-                         *
-                         * If the file size is about the same as page size,
-                         * the entire file can be read and written with a few
-                         * (pipelined) STACK_WINDs, which will be faster
-                         * than "diff" which has to read checksums and then
-                         * read and write.
-                         */
-
-                        algo = sh_algo_from_name (this, "full");
-
-                } else {
-                        algo = sh_algo_from_name (this, "diff");
-                }
-        }
-
-        return algo;
-}
-
-
-int
-afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        struct afr_sh_algorithm *sh_algo = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->algo_completion_cbk = afr_sh_data_erase_pending;
-        sh->algo_abort_cbk      = afr_sh_data_fail;
-
-        sh_algo = afr_sh_data_pick_algo (frame, this);
-
-        sh_algo->fn (frame, this);
-
-        return 0;
-}
-
-int
-afr_sh_data_trim_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-        afr_private_t * priv = NULL;
-        afr_local_t * local  = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-        priv = this->private;
-        local = frame->local;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "ftruncate of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                else
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "ftruncate of %s on subvolume %s completed",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                afr_sh_data_sync_prepare (frame, this);
-
-        return 0;
+__afr_selfheal_data_read_write (call_frame_t *frame, xlator_t *this, fd_t *fd,
+				int source, unsigned char *healed_sinks,
+				off_t offset, size_t size,
+				struct afr_reply *replies)
+{
+	struct iovec *iovec = NULL;
+	int count = 0;
+	struct iobref *iobref = NULL;
+	int ret = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+
+	priv = this->private;
+
+	ret = syncop_readv (priv->children[source], fd, size, offset, 0,
+			    &iovec, &count, &iobref);
+	if (ret <= 0)
+		return ret;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+
+		/*
+		 * TODO: Use fiemap() and discard() to heal holes
+		 * in the future.
+		 *
+		 * For now,
+		 *
+		 * - if the source had any holes at all,
+		 * AND
+		 * - if we are writing past the original file size
+		 *   of the sink
+		 * AND
+		 * - is NOT the last block of the source file. if
+		 *   the block contains EOF, it has to be written
+		 *   in order to set the file size even if the
+		 *   last block is 0-filled.
+		 * AND
+		 * - if the read buffer is filled with only 0's
+		 *
+		 * then, skip writing to this source. We don't depend
+		 * on the write to happen to update the size as we
+		 * have performed an ftruncate() upfront anyways.
+		 */
+#define is_last_block(o,b,s) ((s >= o) && (s <= (o + b)))
+		if (HAS_HOLES ((&replies[source].poststat)) &&
+		    offset >= replies[i].poststat.ia_size &&
+		    !is_last_block (offset, size,
+				    replies[source].poststat.ia_size) &&
+		    (iov_0filled (iovec, count) == 0))
+			continue;
+
+		ret = syncop_writev (priv->children[i], fd, iovec, count,
+				     offset, iobref, 0);
+		if (ret != iov_length (iovec, count)) {
+			/* write() failed on this sink. unset the corresponding
+			   member in sinks[] (which is healed_sinks[] in the
+			   caller) so that this server does NOT get considered
+			   as successfully healed.
+			*/
+			healed_sinks[i] = 0;
+		}
+	}
+	if (iobref)
+		iobref_unref (iobref);
+
+	return ret;
 }
 
 
-int
-afr_sh_data_trim_sinks (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = NULL;
-        afr_local_t * local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int             *sources = NULL;
-        int              call_count = 0;
-        int              i = 0;
-
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sources = sh->sources;
-        call_count = sh->active_sinks;
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] || !local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_trim_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->ftruncate,
-                                   sh->healing_fd, sh->file_size);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
+static int
+afr_selfheal_data_block (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			 int source, unsigned char *healed_sinks, off_t offset,
+			 size_t size, int type, struct afr_reply *replies)
+{
+	int ret = -1;
+	int sink_count = 0;
+	afr_private_t *priv = NULL;
+	unsigned char *data_lock = NULL;
+
+	priv = this->private;
+	sink_count = AFR_COUNT (healed_sinks, priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+
+	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
+				    offset, size, data_lock);
+	{
+		if (ret < sink_count) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		if (type == AFR_SELFHEAL_DATA_DIFF &&
+		    __afr_selfheal_data_checksums_match (frame, this, fd, source,
+							 healed_sinks, offset, size)) {
+			ret = 0;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_data_read_write (frame, this, fd, source,
+						      healed_sinks, offset, size,
+						      replies);
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
+				offset, size, data_lock);
+	return ret;
 }
 
-int
-afr_sh_inode_set_read_ctx (afr_self_heal_t *sh, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        int             ret = 0;
-
-        priv = this->private;
-        sh->source = afr_sh_select_source (sh->sources, priv->child_count);
-        if (sh->source < 0) {
-                ret = -1;
-                goto out;
-        }
 
-        afr_reset_children (sh->fresh_children, priv->child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                sh->fresh_children);
-out:
-        return ret;
-}
 
-int
-afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
+static int
+afr_selfheal_data_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			 unsigned char *healed_sinks)
 {
-        afr_local_t     *local      = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              nsources = 0;
-        int              source = 0;
-        int              i = 0;
-        int              ret = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Pending matrix for: %s",
-                lkowner_utoa (&frame->root->lk_owner));
-
-        nsources = afr_build_sources (this, sh->xattr, sh->buf, sh->pending_matrix,
-                                      sh->sources, sh->success_children,
-                                      AFR_DATA_TRANSACTION, NULL, _gf_false);
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
-
-        if ((nsources == -1)
-            && (priv->favorite_child != -1)
-            && (sh->child_errno[priv->favorite_child] == 0)) {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Picking favorite child %s as authentic source to "
-                        "resolve conflicting data of %s",
-                        priv->children[priv->favorite_child]->name,
-                        local->loc.path);
-
-                sh->sources[priv->favorite_child] = 1;
-
-                nsources = afr_sh_source_count (sh->sources,
-                                                priv->child_count);
-        }
-
-        if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal contents of '%s' (possible "
-                        "split-brain). Please delete the file from all but "
-                        "the preferred subvolume.", local->loc.path);
-
-                local->govinda_gOvinda = 1;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-                afr_sh_data_fail (frame, this);
-                return 0;
-        }
-
-        ret = afr_sh_inode_set_read_ctx (sh, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No active sources found.");
-
-                afr_sh_data_fail (frame, this);
-                return 0;
-        }
-
-        source     = sh->source;
-        sh->block_size = this->ctx->page_size;
-        sh->file_size  = sh->buf[source].ia_size;
-
-        if (FILE_HAS_HOLES (&sh->buf[source]))
-                sh->file_has_holes = 1;
-
-        /* detect changes not visible through pending flags -- JIC */
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == source || sh->child_errno[i])
-                        continue;
-
-                if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
-        }
+	local = frame->local;
+	priv = this->private;
 
-        if (sh->background && sh->unwind) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno);
-                sh->unwound = _gf_true;
-        }
-
-        afr_sh_mark_source_sinks (frame, this);
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no active sinks for performing self-heal on file %s",
-                        local->loc.path);
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
+	AFR_ONLIST (healed_sinks, frame, attr_cbk, fsync, fd, 0, NULL);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "self-healing file %s from subvolume %s to %d other",
-                local->loc.path, priv->children[sh->source]->name,
-                sh->active_sinks);
-        afr_sh_data_trim_sinks (frame, this);
-
-        return 0;
-}
-
-int
-afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
-                                          dict_t **xattr,
-                                          afr_transaction_type txn_type)
-{
-        afr_private_t            *priv      = NULL;
-        int                      read_child = -1;
-        int32_t                  **pending_matrix = NULL;
-        int32_t                  *sources         = NULL;
-        int32_t                  *success_children   = NULL;
-        struct iatt              *bufs            = NULL;
-        int32_t                  nsources         = 0;
-        int32_t                  prev_read_child  = -1;
-        int32_t                  config_read_child = -1;
-        int32_t                  subvol_status = 0;
-
-        priv = this->private;
-        bufs = local->cont.lookup.bufs;
-        success_children = local->cont.lookup.success_children;
-
-        pending_matrix = afr_matrix_create (priv->child_count,
-                                            priv->child_count);
-        if (NULL == pending_matrix)
-                goto out;
-
-        sources = local->cont.lookup.sources;
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-
-        nsources = afr_build_sources (this, xattr, bufs, pending_matrix,
-                                      sources, success_children, txn_type,
-                                      &subvol_status, _gf_false);
-        if (subvol_status & SPLIT_BRAIN)
-                gf_log (this->name, GF_LOG_WARNING, "%s: Possible split-brain",
-                        local->loc.path);
-        if (nsources < 0)
-                goto out;
-
-        prev_read_child = local->read_child_index;
-        config_read_child = priv->read_child;
-        read_child = afr_select_read_child_from_policy (success_children,
-                                                        priv->child_count,
-                                                        prev_read_child,
-                                                        config_read_child,
-                                                        sources);
-out:
-        afr_matrix_cleanup (pending_matrix, priv->child_count);
-        gf_log (this->name, GF_LOG_DEBUG, "returning read_child: %d",
-                read_child);
-        return read_child;
+	for (i = 0; i < priv->child_count; i++)
+		if (healed_sinks[i] && local->replies[i].op_ret != 0)
+			/* fsync() failed. Do NOT consider this server
+			   as successfully healed. Mark it so.
+			*/
+			healed_sinks[i] = 0;
+	return 0;
 }
 
-int
-afr_sh_data_special_file_fix (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_local_t     *local = NULL;
-        int             i = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count ; i++)
-                if (1 == local->child_up[i])
-                        sh->success[i] = 1;
-
-        afr_sh_data_erase_pending (frame, this);
 
-        return 0;
-}
-
-int
-afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       struct iatt *buf)
+static int
+afr_selfheal_data_restore_time (call_frame_t *frame, xlator_t *this,
+				inode_t *inode, int source,
+				unsigned char *healed_sinks,
+				struct afr_reply *replies)
 {
-        afr_private_t   *priv  = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fstat of %s on %s succeeded",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-
-                        sh->buf[child_index] = *buf;
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                }
-        }
-        UNLOCK (&frame->lock);
+	loc_t loc = {0, };
 
-        call_count = afr_frame_return (frame);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        if (call_count == 0) {
-                /* Previous versions of glusterfs might have set
-                 * the pending data xattrs which need to be erased
-                 */
-                if (IA_ISREG (buf->ia_type))
-                        afr_sh_data_fix (frame, this);
-                else
-                        afr_sh_data_special_file_fix (frame, this);
+	AFR_ONLIST (healed_sinks, frame, attr_cbk, setattr, &loc,
+		    &replies[source].poststat,
+		    (GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME), NULL);
 
-        }
+	loc_wipe (&loc);
 
-        return 0;
+	return 0;
 }
 
-
-int
-afr_sh_data_fstat (call_frame_t *frame, xlator_t *this)
+static int
+afr_data_self_heal_type_get (afr_private_t *priv, unsigned char *healed_sinks,
+                             int source, struct afr_reply *replies)
 {
-        afr_self_heal_t *sh    = NULL;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        int call_count = 0;
+        int type = AFR_SELFHEAL_DATA_FULL;
         int i = 0;
 
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        call_count = afr_up_children_count (local->child_up,
-                                            priv->child_count);
-
-        local->call_count = call_count;
-
-        afr_reset_children (sh->success_children, priv->child_count);
-        sh->success_count = 0;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fstat,
-                                           sh->healing_fd);
-
-                        if (!--call_count)
+        if (priv->data_self_heal_algorithm == NULL) {
+                type = AFR_SELFHEAL_DATA_FULL;
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!healed_sinks[i] && i != source)
+                                continue;
+                        if (replies[i].poststat.ia_size) {
+                                type = AFR_SELFHEAL_DATA_DIFF;
                                 break;
+                        }
                 }
+        } else if (strcmp (priv->data_self_heal_algorithm, "full") == 0) {
+                type = AFR_SELFHEAL_DATA_FULL;
+        } else if (strcmp (priv->data_self_heal_algorithm, "diff") == 0) {
+                type = AFR_SELFHEAL_DATA_DIFF;
         }
-
-        return 0;
-}
-
-void
-afr_sh_common_fxattrop_resp_handler (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int32_t op_ret,
-                                     int32_t op_errno, dict_t *xattr)
-{
-        afr_private_t   *priv  = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fxattrop of %s on %s succeeded",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-
-                        sh->xattr[child_index] = dict_ref (xattr);
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                }
-        }
-        UNLOCK (&frame->lock);
-}
-
-int
-afr_post_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xattr)
-{
-        int             call_count  = -1;
-        int             ret = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
-                                             op_errno, xattr);
-
-        local = frame->local;
-        sh = &local->self_heal;
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                (void) afr_build_sources (this, sh->xattr, NULL,
-                                          sh->pending_matrix,
-                                          sh->sources, sh->success_children,
-                                          AFR_DATA_TRANSACTION, NULL, _gf_false);
-                ret = afr_sh_inode_set_read_ctx (sh, this);
-                if (ret)
-                        afr_sh_data_fail (frame, this);
-                else
-                        afr_sh_set_timestamps (frame, this);
-        }
-
-        return 0;
+        return type;
 }
 
-int
-afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                          xlator_t *this, int32_t op_ret, int32_t op_errno,
-                          dict_t *xattr)
-{
-        int call_count  = -1;
-
-        afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
-                                             op_errno, xattr);
-
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                afr_sh_data_fstat (frame, this);
-        }
+static int
+afr_selfheal_data_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int source, unsigned char *healed_sinks,
+		      struct afr_reply *replies)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	off_t off = 0;
+	size_t block = 128 * 1024;
+	int type = AFR_SELFHEAL_DATA_FULL;
+	int ret = -1;
+	call_frame_t *iter_frame = NULL;
+	char *sinks_str = NULL;
+	char *p = NULL;
+
+	priv = this->private;
+
+	sinks_str = alloca0 (priv->child_count * 8);
+	p = sinks_str;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+		p += sprintf (p, "%d ", i);
+	}
+
+	gf_log (this->name, GF_LOG_INFO, "performing data selfheal on %s. "
+		"source=%d sinks=%s",
+		uuid_utoa (fd->inode->gfid), source, sinks_str);
+
+        type = afr_data_self_heal_type_get (priv, healed_sinks, source,
+                                            replies);
+
+	iter_frame = afr_copy_frame (frame);
+	if (!iter_frame)
+		return -ENOMEM;
+
+	for (off = 0; off < replies[source].poststat.ia_size; off += block) {
+		ret = afr_selfheal_data_block (iter_frame, this, fd, source,
+					       healed_sinks, off, block, type,
+					       replies);
+		if (ret < 0)
+			goto out;
+
+		AFR_STACK_RESET (iter_frame);
+	}
+
+	afr_selfheal_data_restore_time (frame, this, fd->inode, source,
+					healed_sinks, replies);
+
+	ret = afr_selfheal_data_fsync (frame, this, fd, healed_sinks);
 
-        return 0;
+out:
+	if (iter_frame)
+		AFR_STACK_DESTROY (iter_frame);
+	return ret;
 }
 
 
-int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this,
-                      afr_fxattrop_cbk_t fxattrop_cbk)
+static int
+__afr_selfheal_truncate_sinks (call_frame_t *frame, xlator_t *this,
+			       fd_t *fd, unsigned char *healed_sinks,
+			       struct afr_reply *replies, uint64_t size)
 {
-        afr_self_heal_t *sh    = NULL;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        dict_t          *xattr_req = NULL;
-        int32_t         *zero_pending = NULL;
-        int call_count = 0;
-        int i = 0;
-        int ret = 0;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        call_count = afr_up_children_count (local->child_up,
-                                            priv->child_count);
-
-        local->call_count = call_count;
-
-        xattr_req = dict_new();
-        if (!xattr_req) {
-                ret = -1;
-                goto out;
-        }
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	unsigned char *larger_sinks = 0;
+	int i = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                zero_pending = GF_CALLOC (3, sizeof (*zero_pending),
-                                          gf_afr_mt_int32_t);
-                if (!zero_pending) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynptr (xattr_req, priv->pending_key[i],
-                                       zero_pending,
-                                       3 * sizeof (*zero_pending));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to set dict value");
-                        goto out;
-                } else {
-                        zero_pending = NULL;
-                }
-        }
+	local = frame->local;
+	priv = this->private;
 
-        afr_reset_xattr (sh->xattr, priv->child_count);
-        afr_reset_children (sh->success_children, priv->child_count);
-        sh->success_count = 0;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, fxattrop_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fxattrop,
-                                           sh->healing_fd, GF_XATTROP_ADD_ARRAY,
-                                           xattr_req);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+	larger_sinks = alloca0 (priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (healed_sinks[i] && replies[i].poststat.ia_size > size)
+			larger_sinks[i] = 1;
+	}
 
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        if (ret) {
-                if (zero_pending)
-                        GF_FREE (zero_pending);
-                sh->op_failed = 1;
-                afr_sh_data_done (frame, this);
-        }
+	AFR_ONLIST (larger_sinks, frame, attr_cbk, ftruncate, fd, size, NULL);
 
-        return 0;
+	for (i = 0; i < priv->child_count; i++)
+		if (healed_sinks[i] && local->replies[i].op_ret == -1)
+			/* truncate() failed. Do NOT consider this server
+			   as successfully healed. Mark it so.
+			*/
+			healed_sinks[i] = 0;
+	return 0;
 }
 
-int
-afr_sh_data_big_lock_success (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->data_lock_held = _gf_true;
-        afr_sh_data_fxattrop (frame, this, afr_sh_data_fxattrop_cbk);
-        return 0;
+/*
+ * If by chance there are multiple sources with differing sizes, select
+ * the largest file as the source.
+ *
+ * This can only happen if data was directly modified in the backend.
+ */
+static int
+__afr_selfheal_data_finalize_source (xlator_t *this, unsigned char *sources,
+				     unsigned char *sinks,
+				     unsigned char *healed_sinks,
+				     unsigned char *locked_on,
+				     struct afr_reply *replies)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	uint64_t size = 0;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int healed_sinks_count = 0;
+
+	priv = this->private;
+
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	healed_sinks_count = AFR_COUNT (healed_sinks, priv->child_count);
+
+	if (locked_count == healed_sinks_count || !sources_count) {
+		/* split brain */
+		return -EIO;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (size <= replies[i].poststat.ia_size) {
+			size = replies[i].poststat.ia_size;
+			source = i;
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (replies[i].poststat.ia_size < size) {
+			sources[i] = 0;
+			sinks[i] = 1;
+		}
+	}
+
+	return source;
 }
 
-int
-afr_sh_data_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
+/*
+ * __afr_selfheal_data_prepare:
+ *
+ * This function inspects the on-disk xattrs and determines which subvols
+ * are sources and sinks.
+ *
+ * The return value is the index of the subvolume to be used as the source
+ * for self-healing, or -1 if no healing is necessary/split brain.
+ */
+static int
+__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     unsigned char *locked_on, unsigned char *sources,
+			     unsigned char *sinks, unsigned char *healed_sinks,
+			     struct afr_reply *replies)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
+	priv = this->private;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Blocking data inodelks "
-                        "failed for %s. by %s",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
+	ret = afr_selfheal_unlocked_discover (frame, fd->inode, fd->inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
 
-                sh->data_lock_failure_handler (frame, this);
-        } else {
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_DATA_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
 
-                gf_log (this->name, GF_LOG_DEBUG, "Blocking data inodelks "
-                        "done for %s by %s. Proceding to self-heal",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
+	source = __afr_selfheal_data_finalize_source (this, sources, sinks,
+						      healed_sinks, locked_on,
+						      replies);
+	if (source < 0)
+		return -EIO;
 
-                sh->data_lock_success_handler (frame, this);
-        }
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
 
-        return 0;
-}
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
 
-int
-afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
-                        "failed for %s. by %s",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
-
-                int_lock->lock_cbk = afr_sh_data_post_blocking_inodelk_cbk;
-                afr_blocking_lock (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
-                        "done for %s by %s. Proceeding to self-heal",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
-                sh->data_lock_success_handler (frame, this);
-        }
-
-        return 0;
+	return source;
 }
 
-int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, off_t start, off_t len)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_DATA_SELF_HEAL_LK;
-
-        afr_set_lock_number (frame, this);
-
-        int_lock->lk_flock.l_start = start;
-        int_lock->lk_flock.l_len   = len;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-        int_lock->lock_cbk         = afr_sh_data_post_nonblocking_inodelk_cbk;
 
-        afr_nonblocking_inodelk (frame, this);
-
-        return 0;
+static int
+__afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		     unsigned char *locked_on)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
+	gf_boolean_t compat = _gf_false;
+	unsigned char *compat_lock = NULL;
+
+	priv = this->private;
+
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+	compat_lock = alloca0 (priv->child_count);
+
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+
+	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name, 0, 0,
+				    data_lock);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_data_prepare (frame, this, fd, data_lock,
+						   sources, sinks, healed_sinks,
+						   locked_replies);
+		if (ret < 0)
+			goto unlock;
+
+		source = ret;
+
+		ret = __afr_selfheal_truncate_sinks (frame, this, fd, healed_sinks,
+						     locked_replies,
+						     locked_replies[source].poststat.ia_size);
+		if (ret < 0)
+			goto unlock;
+
+		ret = 0;
+
+		/* Locking from (LLONG_MAX - 2) to (LLONG_MAX - 1) is for
+		   compatibility with older self-heal clients which do not
+		   hold a lock in the @priv->sh_domain domain to guard
+		   against concurrent ongoing self-heals
+		*/
+		afr_selfheal_inodelk (frame, this, fd->inode, this->name,
+				      LLONG_MAX - 2, 1, compat_lock);
+		compat = _gf_true;
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, fd->inode, this->name, 0, 0,
+				data_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = afr_selfheal_data_do (frame, this, fd, source, healed_sinks,
+				    locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, fd->inode, sources, sinks,
+					 healed_sinks, AFR_DATA_TRANSACTION,
+					 locked_replies, data_lock);
+out:
+	if (compat)
+		afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
+					LLONG_MAX - 2, 1, compat_lock);
+	return ret;
 }
 
-int
-afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_self_heal_t *sh        = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        GF_ASSERT (sh->old_loop_frame);
-        sh_loop_finish (sh->old_loop_frame, this);
-        sh->old_loop_frame = NULL;
-        sh->data_lock_held = _gf_true;
-        afr_sh_data_fxattrop (frame, this, afr_post_sh_data_fxattrop_cbk);
-        return 0;
-}
 
-int
-afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this)
+static fd_t *
+afr_selfheal_data_open (xlator_t *this, inode_t *inode)
 {
-        afr_local_t     *local     = NULL;
-        afr_self_heal_t *sh        = NULL;
+	loc_t loc = {0,};
+	int ret = 0;
+	fd_t *fd = NULL;
 
-        local = frame->local;
-        sh = &local->self_heal;
+	fd = fd_create (inode, 0);
+	if (!fd)
+		return NULL;
 
-        GF_ASSERT (sh->old_loop_frame);
-        sh_loop_finish (sh->old_loop_frame, this);
-        sh->old_loop_frame = NULL;
-        afr_sh_set_timestamps (frame, this);
-        return 0;
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
+	ret = syncop_open (this, &loc, O_RDWR|O_LARGEFILE, fd);
+	if (ret) {
+		fd_unref (fd);
+		fd = NULL;
+	} else {
+		fd_bind (fd);
+	}
 
-int
-afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
-                  afr_lock_cbk_t success_handler,
-                  afr_lock_cbk_t failure_handler)
-{
-        afr_local_t *   local = NULL;
-        afr_self_heal_t * sh  = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
+	loc_wipe (&loc);
 
-        sh->data_lock_success_handler = success_handler;
-        sh->data_lock_failure_handler = failure_handler;
-        return afr_sh_data_lock_rec (frame, this, start, len);
+	return fd;
 }
 
+
 int
-afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd)
+afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        /* TODO: some of the open's might fail.
-           In that case, modify cleanup fn to send flush on those
-           fd's which are already open */
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "open of %s failed on child %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                }
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	int ret = 0;
+	fd_t *fd = NULL;
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "open of %s succeeded on child %s",
-                        local->loc.path,
-                        priv->children[child_index]->name);
-        }
-        UNLOCK (&frame->lock);
+	priv = this->private;
 
-        call_count = afr_frame_return (frame);
+	fd = afr_selfheal_data_open (this, inode);
+	if (!fd)
+		return -EIO;
 
-        if (call_count == 0) {
-                if (sh->op_failed) {
-                        afr_sh_data_fail (frame, this);
-                        return 0;
-                }
+	locked_on = alloca0 (priv->child_count);
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "fd for %s opened, commencing sync",
-                        local->loc.path);
+	ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain, 0, 0,
+				       locked_on);
+	{
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
+		}
 
-                afr_sh_data_lock (frame, this, 0, 0,
-                                  afr_sh_data_big_lock_success,
-                                  afr_sh_data_fail);
-        }
-
-        return 0;
-}
+		ret = __afr_selfheal_data (frame, this, fd, locked_on);
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0, locked_on);
 
+	if (fd)
+		fd_unref (fd);
 
-int
-afr_sh_data_open (call_frame_t *frame, xlator_t *this)
-{
-        int i = 0;
-        int call_count = 0;
-        fd_t *fd = NULL;
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-        local->call_count = call_count;
-
-        fd = fd_create (local->loc.inode, frame->root->pid);
-        sh->healing_fd = fd;
-
-        /* open sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if(!local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_open_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->open,
-                                   &local->loc,
-                                   O_RDWR|O_LARGEFILE, fd, 0);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
-
-int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t *priv = this->private;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        if (sh->do_data_self_heal &&
-            afr_data_self_heal_enabled (priv->data_self_heal)) {
-                afr_sh_data_open (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "not doing data self heal on %s",
-                        local->loc.path);
-                afr_sh_data_done (frame, this);
-        }
-
-        return 0;
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 3aefd3b37..9605d69f4 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1,2412 +1,631 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
-#include "inode.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include "afr-self-heal.h"
 #include "byte-order.h"
-
 #include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-#define AFR_INIT_SH_FRAME_VALS(_frame, _local, _sh, _sh_frame, _sh_local, _sh_sh)\
-        do {\
-                _local = _frame->local;\
-                _sh = &_local->self_heal;\
-                _sh_frame = _sh->sh_frame;\
-                _sh_local = _sh_frame->local;\
-                _sh_sh    = &_sh_local->self_heal;\
-        } while (0);
-
-int
-afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
-                                  int child_index);
-int
-afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->completion_cbk (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_entry_done;
-        afr_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_finish (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-
-        local = frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "finishing entry selfheal of %s", local->loc.path);
-
-        afr_sh_entry_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xattr)
-{
-        long                 i          = 0;
-        int                  call_count = 0;
-        afr_local_t         *local      = NULL;
-        afr_self_heal_t     *sh         = NULL;
-        afr_local_t         *orig_local = NULL;
-        call_frame_t        *orig_frame = NULL;
-        afr_private_t       *priv       = NULL;
-        int32_t             read_child  = -1;
-
-        local = frame->local;
-        priv  = this->private;
-        sh = &local->self_heal;
-        i = (long)cookie;
-
-
-        afr_children_add_child (sh->fresh_children, i, priv->child_count);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to erase pending xattrs on %s (%s)",
-                        local->loc.path, priv->children[i]->name,
-                        strerror (op_errno));
-        }
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->source == -1) {
-                        //this happens if the forced merge option is set
-                        read_child = sh->fresh_children[0];
-                } else {
-                        read_child = sh->source;
-                }
-                afr_inode_set_read_ctx (this, sh->inode, read_child,
-                                        sh->fresh_children);
-                orig_frame = sh->orig_frame;
-                orig_local = orig_frame->local;
-
-                if (sh->source != -1) {
-                        orig_local->cont.lookup.buf.ia_nlink = sh->buf[sh->source].ia_nlink;
-                }
-
-                afr_sh_entry_finish (frame, this);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-        int              need_unwind = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (sh->entries_skipped) {
-                need_unwind = 1;
-                sh->op_failed = _gf_true;
-                goto out;
-        }
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_ENTRY_TRANSACTION);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        if (call_count == 0)
-                need_unwind = 1;
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_ENTRY_TRANSACTION);
-
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_entry_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &local->loc,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
-out:
-        if (need_unwind)
-                afr_sh_entry_finish (frame, this);
-
-        return 0;
-}
-
 
 
 static int
-next_active_source (call_frame_t *frame, xlator_t *this,
-                    int current_active_source)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              source = -1;
-        int              next_active_source = -1;
-        int              i = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        source = sh->source;
-
-        if (source != -1) {
-                if (current_active_source != source)
-                        next_active_source = source;
-                goto out;
-        }
-
-        /*
-          the next active sink becomes the source for the
-          'conservative decision' of merging all entries
-        */
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((sh->sources[i] == 0)
-                    && (local->child_up[i] == 1)
-                    && (i > current_active_source)) {
-
-                        next_active_source = i;
-                        break;
-                }
-        }
-out:
-        return next_active_source;
-}
-
-
-
-static int
-next_active_sink (call_frame_t *frame, xlator_t *this,
-                  int current_active_sink)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              next_active_sink = -1;
-        int              i = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        /*
-          the next active sink becomes the source for the
-          'conservative decision' of merging all entries
-        */
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((sh->sources[i] == 0)
-                    && (local->child_up[i] == 1)
-                    && (i > current_active_sink)) {
-
-                        next_active_sink = i;
-                        break;
-                }
-        }
-
-        return next_active_sink;
-}
-
-int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-                             int active_src);
-
-int
-afr_sh_entry_expunge_entry_done (call_frame_t *frame, xlator_t *this,
-                                 int active_src, int32_t op_ret,
-                                 int32_t op_errno)
-{
-        int              call_count = 0;
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                afr_sh_entry_expunge_subvol (frame, this, active_src);
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_parent_setattr_cbk (call_frame_t *expunge_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop)
-{
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh    = NULL;
-        call_frame_t    *frame         = NULL;
-        int              active_src    = (long) cookie;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh    = &expunge_local->self_heal;
-        frame         = expunge_sh->sh_frame;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "setattr on parent directory of %s on subvolume %s failed: %s",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name, strerror (op_errno));
-        }
-
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
-                                 xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              active_src = 0;
-        int32_t          valid = 0;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-
-        active_src = (long) cookie;
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "removed %s on %s",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "removing %s on %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-        }
-
-        valid = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_parent_setattr_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->setattr,
-                           &expunge_sh->parent_loc,
-                           &expunge_sh->parentbuf,
-                           valid);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_unlink (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "expunging file %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->unlink,
-                           &expunge_local->loc);
-
-        return 0;
-}
-
-
-
-int
-afr_sh_entry_expunge_rmdir (call_frame_t *expunge_frame, xlator_t *this,
-                            int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "expunging directory %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
+afr_selfheal_entry_delete (call_frame_t *frame, xlator_t *this, inode_t *dir,
+			   const char *name, inode_t *inode, int child,
+			   struct afr_reply *replies)
+{
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+	char g[64];
+
+	priv = this->private;
+
+	subvol = priv->children[child];
+
+	loc.parent = inode_ref (dir);
+	uuid_copy (loc.pargfid, dir->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
+
+	if (replies[child].valid && replies[child].op_ret == 0) {
+		switch (replies[child].poststat.ia_type) {
+		case IA_IFDIR:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging dir %s/%s (%s) on %s",
+				uuid_utoa (dir->gfid), name,
+				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
+				subvol->name);
+			ret = syncop_rmdir (subvol, &loc, 1);
+			break;
+		default:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging file %s/%s (%s) on %s",
+				uuid_utoa (dir->gfid), name,
+				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
+				subvol->name);
+			ret = syncop_unlink (subvol, &loc);
+			break;
+		}
+	}
+
+	loc_wipe (&loc);
+
+	return ret;
+}
+
+
+int
+afr_selfheal_recreate_entry (call_frame_t *frame, xlator_t *this, int dst,
+			     int source, inode_t *dir, const char *name,
+			     inode_t *inode, struct afr_reply *replies)
+{
+	int ret = 0;
+	loc_t loc = {0,};
+	loc_t srcloc = {0,};
+	afr_private_t *priv = NULL;
+	dict_t *xdata = NULL;
+	struct iatt *iatt = NULL;
+	char *linkname = NULL;
+	mode_t mode = 0;
+	struct iatt newent = {0,};
+
+	priv = this->private;
+
+	xdata = dict_new();
+	if (!xdata)
+		return -ENOMEM;
+
+	loc.parent = inode_ref (dir);
+	uuid_copy (loc.pargfid, dir->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
+
+	ret = afr_selfheal_entry_delete (frame, this, dir, name, inode, dst,
+					 replies);
+	if (ret)
+		goto out;
+
+	ret = dict_set_static_bin (xdata, "gfid-req",
+				   replies[source].poststat.ia_gfid, 16);
+	if (ret)
+		goto out;
+
+	iatt = &replies[source].poststat;
+
+	srcloc.inode = inode_ref (inode);
+	uuid_copy (srcloc.gfid, iatt->ia_gfid);
+
+	mode = st_mode_from_ia (iatt->ia_prot, iatt->ia_type);
+
+	switch (iatt->ia_type) {
+	case IA_IFDIR:
+		ret = syncop_mkdir (priv->children[dst], &loc, mode, xdata, 0);
+		break;
+	case IA_IFLNK:
+		ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0);
+		if (ret == 0) {
+			ret = syncop_link (priv->children[dst], &srcloc, &loc);
+		} else {
+			ret = syncop_readlink (priv->children[source], &srcloc,
+					       &linkname, 4096);
+			if (ret <= 0)
+				goto out;
+			ret = syncop_symlink (priv->children[dst], &loc, linkname,
+					      xdata, NULL);
+		}
+		break;
+	default:
+		ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+		if (ret)
+			goto out;
+		ret = syncop_mknod (priv->children[dst], &loc, mode,
+				    iatt->ia_rdev, xdata, &newent);
+		if (ret == 0 && iatt->ia_size && !newent.ia_size) {
+			/* New entry created. Mark @dst pending on all sources */
+			ret = 1;
+		}
+		break;
+	}
 
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->rmdir,
-                           &expunge_local->loc, 1);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src, struct iatt *buf,
-                             struct iatt *parentbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        int              type = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-        loc_t           *loc           = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        local         = frame->local;
-        sh            = &local->self_heal;
-        loc           = &expunge_local->loc;
-
-        type = buf->ia_type;
-        if (loc->parent && uuid_is_null (loc->parent->gfid))
-                uuid_copy (loc->pargfid, parentbuf->ia_gfid);
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-                afr_sh_entry_expunge_unlink (expunge_frame, this, active_src);
-                break;
-        case IA_IFDIR:
-                afr_sh_entry_expunge_rmdir (expunge_frame, this, active_src);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                goto out;
-                break;
-        }
-
-        return 0;
 out:
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, -1, EINVAL);
-
-        return 0;
+	if (xdata)
+		dict_unref (xdata);
+	loc_wipe (&loc);
+	loc_wipe (&srcloc);
+	return ret;
 }
 
 
-int
-afr_sh_entry_expunge_lookup_cbk (call_frame_t *expunge_frame, void *cookie,
-                                 xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno,
-                                 inode_t *inode, struct iatt *buf, dict_t *x,
-                                 struct iatt *postparent)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        int              active_src = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        active_src = (long) cookie;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "lookup of %s on %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        afr_sh_entry_expunge_remove (expunge_frame, this, active_src, buf,
-                                     postparent);
-
-        return 0;
-out:
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_purge (call_frame_t *expunge_frame, xlator_t *this,
-                            int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "looking up %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_lookup_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->lookup,
-                           &expunge_local->loc, 0);
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_entry_cbk (call_frame_t *expunge_frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *buf, dict_t *x,
-                                struct iatt *postparent)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              source = 0;
-        call_frame_t    *frame = NULL;
-        int              active_src = 0;
-        int              need_expunge = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        active_src = expunge_sh->active_source;
-        source = (long) cookie;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret == -1 && op_errno == ENOENT)
-                need_expunge = 1;
-        else if (op_ret == -1)
-                goto out;
-
-        if (!uuid_is_null (expunge_sh->entrybuf.ia_gfid) &&
-            !uuid_is_null (buf->ia_gfid) &&
-            (uuid_compare (expunge_sh->entrybuf.ia_gfid, buf->ia_gfid) != 0)) {
-                char uuidbuf1[64];
-                char uuidbuf2[64];
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "entry %s found on %s with mismatching gfid (%s/%s)",
-                        expunge_local->loc.path,
-                        priv->children[source]->name,
-                        uuid_utoa_r (expunge_sh->entrybuf.ia_gfid, uuidbuf1),
-                        uuid_utoa_r (buf->ia_gfid, uuidbuf2));
-                need_expunge = 1;
-        }
-
-        if (need_expunge) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "missing entry %s on %s",
-                        expunge_local->loc.path,
-                        priv->children[source]->name);
-
-                if (postparent)
-                        expunge_sh->parentbuf = *postparent;
-
-                afr_sh_entry_expunge_purge (expunge_frame, this, active_src);
-
-                return 0;
-        }
-
-out:
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s exists under %s",
-                        expunge_local->loc.path,
-                        priv->children[source]->name);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "looking up %s under %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[source]->name,
-                        strerror (op_errno));
-        }
-
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
-                            gf_dirent_t *entry)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              ret = -1;
-        call_frame_t    *expunge_frame = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              active_src = 0;
-        int              source = 0;
-        int              op_errno = 0;
-        char            *name = NULL;
-        int             op_ret = -1;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-        source = sh->source;
-        sh->expunge_done = afr_sh_entry_expunge_entry_done;
-
-        name = entry->d_name;
-
-        if ((strcmp (name, ".") == 0)
-            || (strcmp (name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        name, local->loc.path);
-                op_ret = 0;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "inspecting existence of %s under %s",
-                name, local->loc.path);
-
-        expunge_frame = copy_frame (frame);
-        if (!expunge_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
-
-        expunge_frame->local = expunge_local;
-        expunge_sh = &expunge_local->self_heal;
-        expunge_sh->sh_frame = frame;
-        expunge_sh->active_source = active_src;
-        expunge_sh->entrybuf = entry->d_stat;
-        loc_copy (&expunge_sh->parent_loc, &local->loc);
-
-        ret = afr_build_child_loc (this, &expunge_local->loc, &local->loc,
-                                   name);
-        if (ret != 0) {
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "looking up %s on %s", expunge_local->loc.path,
-                priv->children[source]->name);
-
-        STACK_WIND_COOKIE (expunge_frame,
-                           afr_sh_entry_expunge_entry_cbk,
-                           (void *) (long) source,
-                           priv->children[source],
-                           priv->children[source]->fops->lookup,
-                           &expunge_local->loc, 0);
-
-        ret = 0;
-out:
-        if (ret == -1)
-                sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_readdir_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        gf_dirent_t     *entry = NULL;
-        off_t            last_offset = 0;
-        int              active_src = 0;
-        int              entry_count = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "readdir of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[active_src]->name,
-                                strerror (op_errno));
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "readdir of %s on subvolume %s complete",
-                                local->loc.path,
-                                priv->children[active_src]->name);
-                }
-
-                afr_sh_entry_expunge_all (frame, this);
-                return 0;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-                entry_count++;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "readdir'ed %d entries from %s",
-                entry_count, priv->children[active_src]->name);
-
-        sh->offset = last_offset;
-        local->call_count = entry_count;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                afr_sh_entry_expunge_entry (frame, this, entry);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-                             int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        STACK_WIND (frame, afr_sh_entry_expunge_readdir_cbk,
-                    priv->children[active_src],
-                    priv->children[active_src]->fops->readdirp,
-                    sh->healing_fd, sh->block_size, sh->offset, NULL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this)
+static int
+afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			    int source, struct afr_reply *replies,
+			    unsigned char *sources, unsigned char *newentry)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              active_src = -1;
+	int ret = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	dict_t *xattr = NULL;
+	int **changelog = NULL;
+	int idx = 0;
 
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
+	priv = this->private;
 
-        sh->offset = 0;
+	idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
 
-        if (sh->source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sources for %s to expunge entries",
-                        local->loc.path);
-                goto out;
-        }
+	uuid_copy (inode->gfid, replies[source].poststat.ia_gfid);
 
-        active_src = next_active_sink (frame, this, sh->active_source);
-        sh->active_source = active_src;
+	changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
 
-        if (sh->op_failed) {
-                goto out;
-        }
+	xattr = dict_new();
+	if (!xattr)
+		return -ENOMEM;
 
-        if (active_src == -1) {
-                /* completed creating missing files on all subvolumes */
-                goto out;
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (!newentry[i])
+			continue;
+		changelog[i][idx] = hton32(1);
+	}
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "expunging entries of %s on %s to other sinks",
-                local->loc.path, priv->children[active_src]->name);
+	afr_set_pending_dict (priv, xattr, changelog);
 
-        afr_sh_entry_expunge_subvol (frame, this, active_src);
-
-        return 0;
-out:
-        afr_sh_entry_impunge_all (frame, this);
-        return 0;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		afr_selfheal_post_op (frame, this, inode, i, xattr);
+	}
 
+	dict_unref (xattr);
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_entry_done (call_frame_t *frame, xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno)
-{
-        int              call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        if (op_ret < 0)
-                sh->entries_skipped = _gf_true;
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                afr_sh_entry_impunge_subvol (frame, this);
-
-        return 0;
-}
-
-void
-afr_sh_entry_call_impunge_done (call_frame_t *impunge_frame, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *impunge_local = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-
-        AFR_STACK_DESTROY (impunge_frame);
-        sh->impunge_done (frame, this, op_ret, op_errno);
-}
-
-int
-afr_sh_entry_impunge_setattr_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  struct iatt *preop, struct iatt *postop)
-{
-        int              call_count = 0;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              child_index = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        child_index = (long) cookie;
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setattr done for %s on %s",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "setattr (%s) on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-        }
-
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                0, op_errno);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_entry_impunge_parent_setattr_cbk (call_frame_t *setattr_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop)
-{
-        int             call_count = 0;
-        afr_local_t     *setattr_local = NULL;
-
-        setattr_local = setattr_frame->local;
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "setattr on parent directory (%s) failed: %s",
-                        setattr_local->loc.path, strerror (op_errno));
-        }
-
-        call_count = afr_frame_return (setattr_frame);
-        if (call_count == 0)
-                AFR_STACK_DESTROY (setattr_frame);
-        return 0;
+static int
+__afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			    char *name, inode_t *inode, int source,
+			    unsigned char *sources, unsigned char *healed_sinks,
+			    unsigned char *locked_on, struct afr_reply *replies)
+{
+	int ret = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	unsigned char *newentry = NULL;
+
+	priv = this->private;
+	newentry = alloca0 (priv->child_count);
+
+	if (!replies[source].valid)
+		return -EIO;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+		if (replies[source].op_ret == -1 &&
+		    replies[source].op_errno == ENOENT) {
+			ret = afr_selfheal_entry_delete (frame, this, fd->inode,
+							 name, inode, i, replies);
+		} else {
+			if (!uuid_compare (replies[i].poststat.ia_gfid,
+					   replies[source].poststat.ia_gfid))
+				continue;
+
+			ret = afr_selfheal_recreate_entry (frame, this, i, source,
+							   fd->inode, name, inode,
+							   replies);
+			if (ret > 0) {
+				newentry[i] = 1;
+				ret = 0;
+			}
+		}
+		if (ret < 0)
+			break;
+	}
+
+	if (AFR_COUNT (newentry, priv->child_count))
+		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
+					    sources, newentry);
+	return ret;
 }
 
-int
-afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_local_t     *setattr_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *setattr_frame = NULL;
-        int32_t          valid = 0;
-        int32_t          op_errno = 0;
-        int              child_index = 0;
-        int              call_count = 0;
-        int              i = 0;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setting ownership of %s on %s to %d/%d",
-                impunge_local->loc.path,
-                priv->children[child_index]->name,
-                impunge_sh->entrybuf.ia_uid,
-                impunge_sh->entrybuf.ia_gid);
-
-        setattr_frame = copy_frame (impunge_frame);
-        if (!setattr_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-        AFR_LOCAL_ALLOC_OR_GOTO (setattr_frame->local, out);
-        setattr_local = setattr_frame->local;
-        call_count = afr_errno_count (NULL, impunge_sh->child_errno,
-                                      priv->child_count, 0);
-        loc_copy (&setattr_local->loc, &impunge_sh->parent_loc);
-        impunge_local->call_count = call_count;
-        setattr_local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (impunge_sh->child_errno[i])
-                        continue;
-                valid         = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-                STACK_WIND_COOKIE (setattr_frame,
-                                   afr_sh_entry_impunge_parent_setattr_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &setattr_local->loc,
-                                   &impunge_sh->parentbuf, valid);
-
-                valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                        GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-                STACK_WIND_COOKIE (impunge_frame,
-                                   afr_sh_entry_impunge_setattr_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &impunge_local->loc,
-                                   &impunge_sh->entrybuf, valid);
-                call_count--;
-        }
-        GF_ASSERT (!call_count);
-        return 0;
-out:
-        if (setattr_frame)
-                AFR_STACK_DESTROY (setattr_frame);
-        afr_sh_entry_call_impunge_done (impunge_frame, this, 0, op_errno);
-        return 0;
-}
 
-int
-afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  dict_t *xattr)
+static int
+__afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     char *name, inode_t *inode, unsigned char *sources,
+			     unsigned char *healed_sinks, unsigned char *locked_on,
+			     struct afr_reply *replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              child_index = 0;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to perform xattrop on %s (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        afr_sh_entry_impunge_setattr (impunge_frame, this);
-        return 0;
-out:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        -1, op_errno);
-        return 0;
-}
+	int ret = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int source = -1;
 
-void
-afr_sh_prepare_new_entry_pending_matrix (int32_t **pending,
-                                         int *child_errno,
-                                         struct iatt *buf,
-                                         unsigned int child_count)
-{
-        int              midx             = 0;
-        int              idx              = 0;
-        int              i                = 0;
-
-        midx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-        if (IA_ISDIR (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-        else if (IA_ISREG (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        else
-                idx = -1;
-        for (i = 0; i < child_count; i++) {
-                if (child_errno[i])
-                        continue;
-                pending[i][midx] = hton32 (1);
-                if (idx == -1)
-                        continue;
-                pending[i][idx] = hton32 (1);
-        }
-}
+	priv = this->private;
 
-int
-afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
-                                      xlator_t *this)
-{
-        int              active_src       = 0;
-        dict_t          *xattr            = NULL;
-        afr_private_t   *priv             = NULL;
-        afr_local_t     *impunge_local    = NULL;
-        afr_self_heal_t *impunge_sh       = NULL;
-        int32_t         op_errno          = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-
-        afr_sh_prepare_new_entry_pending_matrix (impunge_local->pending,
-                                                 impunge_sh->child_errno,
-                                                 &impunge_sh->entrybuf,
-                                                 priv->child_count);
-        xattr = dict_new ();
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        afr_set_pending_dict (priv, xattr, impunge_local->pending);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->xattrop,
-                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr);
-
-        if (xattr)
-                dict_unref (xattr);
-        return 0;
-out:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        -1, op_errno);
-        return 0;
-}
+	for (i = 0; i < priv->child_count; i++) {
+		if (replies[i].valid && replies[i].op_ret == 0) {
+			source = i;
+			break;
+		}
+	}
 
-int
-afr_sh_entry_impunge_newfile_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  inode_t *inode, struct iatt *stbuf,
-                                  struct iatt *preparent,
-                                  struct iatt *postparent)
-{
-        int              call_count       = 0;
-        afr_private_t   *priv             = NULL;
-        afr_local_t     *impunge_local    = NULL;
-        afr_self_heal_t *impunge_sh       = NULL;
-        int              child_index      = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                impunge_sh->child_errno[child_index] = op_errno;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "creation of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-        } else {
-                impunge_sh->child_errno[child_index] = 0;
-        }
-
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0) {
-                if (!afr_errno_count (NULL, impunge_sh->child_errno,
-                                      priv->child_count, 0)) {
-                        // new_file creation failed every where
-                        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                        -1, op_errno);
-                        goto out;
-                }
-                afr_sh_entry_impunge_perform_xattrop (impunge_frame, this);
-        }
-out:
-        return 0;
-}
+	if (source == -1) {
+		/* entry got deleted in the mean time? */
+		return 0;
+	}
 
-int
-afr_sh_entry_impunge_hardlink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                   xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, inode_t *inode,
-                                   struct iatt *buf, struct iatt *preparent,
-                                   struct iatt *postparent)
-{
-        int              call_count        = 0;
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
+	for (i = 0; i < priv->child_count; i++) {
+		if (i == source || !healed_sinks[i])
+			continue;
 
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
+		if (replies[i].op_errno != ENOENT)
+			continue;
 
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0)
-                sh->impunge_done (frame, this, op_ret, op_errno);
+		ret = afr_selfheal_recreate_entry (frame, this, i, source,
+						   fd->inode, name, inode,
+						   replies);
+	}
 
-        return 0;
+	return ret;
 }
 
-int
-afr_sh_entry_impunge_hardlink (call_frame_t *impunge_frame, xlator_t *this,
-                               int child_index)
-{
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh  = NULL;
-        loc_t           *loc           = NULL;
-        struct iatt     *buf           = NULL;
-        loc_t            oldloc        = {0};
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        loc = &impunge_local->loc;
-        buf = &impunge_sh->entrybuf;
-
-        oldloc.inode = inode_ref (loc->inode);
-        uuid_copy (oldloc.gfid, buf->ia_gfid);
-        gf_log (this->name, GF_LOG_DEBUG, "linking missing file %s on %s",
-                loc->path, priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_hardlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->link,
-                           &oldloc, loc);
-        loc_wipe (&oldloc);
-
-        return 0;
-}
 
-int
-afr_sh_nameless_lookup_cbk (call_frame_t *impunge_frame, void *cookie,
-                            xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xattr,
-                            struct iatt *postparent)
-{
-        if (op_ret < 0) {
-                 afr_sh_entry_impunge_create_file (impunge_frame, this,
-                                                   (long)cookie);
-        } else {
-                afr_sh_entry_impunge_hardlink (impunge_frame, this,
-                                               (long)cookie);
-        }
-        return 0;
-}
-
-int
-afr_sh_entry_impunge_check_hardlink (call_frame_t *impunge_frame,
-                                     xlator_t *this,
-                                     int child_index, struct iatt *stbuf)
+static int
+__afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     char *name, inode_t *inode, int source,
+			     unsigned char *sources, unsigned char *healed_sinks,
+			     unsigned char *locked_on, struct afr_reply *replies)
 {
-        afr_private_t   *priv          = NULL;
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        loc_t           *loc           = NULL;
-        dict_t          *xattr_req     = NULL;
-        loc_t            oldloc        = {0};
-        int              ret           = -1;
-
-        priv = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        loc = &impunge_local->loc;
-
-        xattr_req = dict_new ();
-        if (!xattr_req)
-                goto out;
-        oldloc.inode = inode_ref (loc->inode);
-        uuid_copy (oldloc.gfid, stbuf->ia_gfid);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_nameless_lookup_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->lookup,
-                           &oldloc, xattr_req);
-        ret = 0;
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        loc_wipe (&oldloc);
-        if (ret)
-                sh->impunge_done (frame, this, -1, ENOMEM);
-        return 0;
-}
+	int ret = -1;
 
-int
-afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
-                            int child_index, struct iatt *stbuf)
-{
-        afr_private_t *priv          = NULL;
-        afr_local_t   *impunge_local = NULL;
-        dict_t        *dict          = NULL;
-        int            ret           = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing file %s on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
-
-        dict = dict_new ();
-        if (!dict)
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-
-        GF_ASSERT (!uuid_is_null (stbuf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
-                        impunge_local->loc.path);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->mknod,
-                           &impunge_local->loc,
-                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           makedev (ia_major (stbuf->ia_rdev),
-                                    ia_minor (stbuf->ia_rdev)), dict);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
+	if (source < 0)
+		ret = __afr_selfheal_merge_dirent (frame, this, fd, name, inode,
+						   sources, healed_sinks,
+						   locked_on, replies);
+	else
+		ret = __afr_selfheal_heal_dirent (frame, this, fd, name, inode,
+						  source, sources, healed_sinks,
+						  locked_on, replies);
+	return ret;
 }
 
 
-
-int
-afr_sh_entry_impunge_mkdir (call_frame_t *impunge_frame, xlator_t *this,
-                            int child_index, struct iatt *stbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        dict_t          *dict = NULL;
-
-        int ret = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                return 0;
-        }
-
-        GF_ASSERT (!uuid_is_null (stbuf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
-                        impunge_local->loc.path);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing directory %s on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->mkdir,
-                           &impunge_local->loc,
-                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           dict);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
+static int
+afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			   int source, unsigned char *sources,
+			   unsigned char *healed_sinks, char *name)
+{
+	afr_private_t *priv = NULL;
+	int ret = 0;
+	unsigned char *locked_on = NULL;
+	struct afr_reply *replies = NULL;
+	inode_t *inode = NULL;
+
+	priv = this->private;
+
+	locked_on = alloca0 (priv->child_count);
+
+	replies = alloca0 (priv->child_count * sizeof(*replies));
+
+	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name,
+				    name, locked_on);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		inode = afr_selfheal_unlocked_lookup_on (frame, fd->inode, name,
+							 replies, locked_on);
+		if (!inode) {
+			ret = -ENOMEM;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_entry_dirent (frame, this, fd, name, inode,
+						   source, sources, healed_sinks,
+						   locked_on, replies);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, name,
+				locked_on);
+	if (inode)
+		inode_unref (inode);
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
-                              int child_index, const char *linkname)
+static int
+afr_selfheal_entry_do_subvol (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			      int child, int source, unsigned char *sources,
+			      unsigned char *healed_sinks)
 {
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
-        dict_t          *dict          = NULL;
-        struct iatt     *buf           = NULL;
-        int              ret           = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        buf = &impunge_local->cont.symlink.buf;
-
-        dict = dict_new ();
-        if (!dict) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                -1, ENOMEM);
-                goto out;
-        }
-
-        GF_ASSERT (!uuid_is_null (buf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, buf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: dict set gfid failed",
-                        impunge_local->loc.path);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing symlink %s -> %s on %s",
-                impunge_local->loc.path, linkname,
-                priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->symlink,
-                           linkname, &impunge_local->loc, dict);
-
-        if (dict)
-                dict_unref (dict);
-out:
-        return 0;
-}
+	int ret = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	off_t offset = 0;
+	call_frame_t *iter_frame = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
 
+	priv = this->private;
+	subvol = priv->children[child];
 
-int
-afr_sh_entry_impunge_symlink_unlink_cbk (call_frame_t *impunge_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preparent,
-                                         struct iatt *postparent)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "unlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        afr_sh_entry_impunge_symlink (impunge_frame, this, child_index,
-                                      impunge_sh->linkname);
-
-        return 0;
-out:
-        LOCK (&impunge_frame->lock);
-        {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
+	INIT_LIST_HEAD (&entries.list);
 
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
+	iter_frame = afr_copy_frame (frame);
+	if (!iter_frame)
+		return -ENOMEM;
 
-        return 0;
-}
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
 
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
 
-int
-afr_sh_entry_impunge_symlink_unlink (call_frame_t *impunge_frame, xlator_t *this,
-                                     int child_index)
-{
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
+			if (__is_root_gfid (fd->inode->gfid) &&
+			    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR))
+				continue;
 
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
+			ret = afr_selfheal_entry_dirent (iter_frame, this, fd,
+							 source, sources,
+							 healed_sinks,
+							 entry->d_name);
+			AFR_STACK_RESET (iter_frame);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "unlinking symlink %s with wrong target on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
+			if (ret)
+				break;
+		}
 
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_symlink_unlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->unlink,
-                           &impunge_local->loc);
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
 
-        return 0;
+	AFR_STACK_DESTROY (iter_frame);
+	return ret;
 }
 
-
-int
-afr_sh_entry_impunge_readlink_sink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                        xlator_t *this,
-                                        int32_t op_ret, int32_t op_errno,
-                                        const char *linkname, struct iatt *sbuf)
+static int
+afr_selfheal_entry_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		       int source, unsigned char *sources,
+		       unsigned char *healed_sinks,
+		       struct afr_reply *locked_replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-        int              active_src = -1;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-        active_src    = impunge_sh->active_source;
-
-        child_index = (long) cookie;
-
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "readlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        /* symlink doesn't exist on the sink */
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                afr_sh_entry_impunge_symlink (impunge_frame, this,
-                                              child_index, impunge_sh->linkname);
-                return 0;
-        }
-
-
-        /* symlink exists on the sink, so check if targets match */
-
-        if (strcmp (linkname, impunge_sh->linkname) == 0) {
-                /* targets match, nothing to do */
-
-                goto out;
-        } else {
-                /*
-                 * Hah! Sneaky wolf in sheep's clothing!
-                 */
-                afr_sh_entry_impunge_symlink_unlink (impunge_frame, this,
-                                                     child_index);
-                return 0;
-        }
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int ret = 0;
 
-out:
-        LOCK (&impunge_frame->lock);
-        {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
+	priv = this->private;
 
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
+	gf_log (this->name, GF_LOG_INFO, "performing entry selfheal on %s",
+		uuid_utoa (fd->inode->gfid));
 
-        return 0;
+	for (i = 0; i < priv->child_count; i++) {
+		if (i != source && !healed_sinks[i])
+			continue;
+		ret = afr_selfheal_entry_do_subvol (frame, this, fd, i, source,
+						    sources, healed_sinks);
+		if (ret)
+			break;
+	}
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_readlink_sink (call_frame_t *impunge_frame, xlator_t *this,
-                                    int child_index)
+static int
+__afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
+				      unsigned char *sinks,
+				      unsigned char *locked_on,
+				      struct afr_reply *replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "checking symlink target of %s on %s",
-                impunge_local->loc.path, priv->children[child_index]->name);
+	priv = this->private;
 
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_sink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->readlink,
-                           &impunge_local->loc, 4096);
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
 
-        return 0;
-}
+	if (locked_count == sinks_count || !sources_count) {
+		return -1;
+	}
 
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i]) {
+			source = i;
+			break;
+		}
+	}
 
-int
-afr_sh_entry_impunge_readlink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   const char *linkname, struct iatt *sbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-        int              active_src = -1;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "readlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        impunge_sh->linkname = gf_strdup (linkname);
-        afr_sh_entry_impunge_readlink_sink (impunge_frame, this, child_index);
-
-        return 0;
-
-out:
-        LOCK (&impunge_frame->lock);
-        {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
-
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
-
-        return 0;
+	return source;
 }
 
 
-int
-afr_sh_entry_impunge_readlink (call_frame_t *impunge_frame, xlator_t *this,
-                               int child_index, struct iatt *stbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              active_src = -1;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-        impunge_local->cont.symlink.buf = *stbuf;
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->readlink,
-                           &impunge_local->loc, 4096);
-
-        return 0;
-}
-
-int
-afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
-                             int child_index)
-{
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       type = IA_INVAL;
-        int             active_src = 0;
-        struct iatt     *buf = NULL;
-
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        afr_update_loc_gfids (&impunge_local->loc, &impunge_sh->entrybuf,
-                              &impunge_sh->parentbuf);
-
-        buf = &impunge_sh->entrybuf;
-        type = buf->ia_type;
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-                afr_sh_entry_impunge_check_hardlink (impunge_frame, this,
-                                                     child_index, buf);
-                break;
-        case IA_IFDIR:
-                afr_sh_entry_impunge_mkdir (impunge_frame, this,
-                                            child_index, buf);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                sh->impunge_done (frame, this, -1, EINVAL);
-                break;
-        }
-
-        return 0;
-}
-
-int
-afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
-                                  int child_index)
-{
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       type = IA_INVAL;
-        int             active_src = 0;
-        struct iatt     *buf = NULL;
-
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        buf = &impunge_sh->entrybuf;
-        type = buf->ia_type;
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-                afr_sh_entry_impunge_mknod (impunge_frame, this,
-                                            child_index, buf);
-                break;
-        case IA_IFLNK:
-                afr_sh_entry_impunge_readlink (impunge_frame, this,
-                                               child_index, buf);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                sh->impunge_done (frame, this, -1, EINVAL);
-                break;
-        }
-
-        return 0;
-}
-
-gf_boolean_t
-afr_sh_need_recreate (afr_self_heal_t *impunge_sh, unsigned int child,
-                      unsigned int child_count)
+static int
+__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			      unsigned char *locked_on, unsigned char *sources,
+			      unsigned char *sinks, unsigned char *healed_sinks,
+			      struct afr_reply *replies, int *source_p)
 {
-        gf_boolean_t    recreate = _gf_false;
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-        GF_ASSERT (impunge_sh->child_errno);
+	priv = this->private;
 
-        if (child == impunge_sh->active_source)
-                goto out;
+	ret = afr_selfheal_unlocked_discover (frame, fd->inode, fd->inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
 
-        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
-                recreate = _gf_true;
-                goto out;
-        }
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_ENTRY_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
 
-        if (impunge_sh->child_errno[child] == ENOENT)
-                recreate = _gf_true;
-out:
-        return recreate;
-}
+	source = __afr_selfheal_entry_finalize_source (this, sources, sinks,
+						       locked_on, replies);
+	if (source < 0) {
+		/* If source is < 0 (typically split-brain), we perform a
+		   conservative merge of entries rather than erroring out */
+	}
+	*source_p = source;
 
-unsigned int
-afr_sh_recreate_count (afr_self_heal_t *impunge_sh, int *sources,
-                       unsigned int child_count)
-{
-        int             count = 0;
-        int             i = 0;
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
 
-        for (i = 0; i < child_count; i++) {
-                if (afr_sh_need_recreate (impunge_sh, i, child_count))
-                        count++;
-        }
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
 
-        return count;
+	return ret;
 }
 
-int
-afr_sh_entry_call_impunge_recreate (call_frame_t *impunge_frame,
-                                    xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        unsigned int     recreate_count = 0;
-        int              i = 0;
-        int              active_src = 0;
-
-        priv          = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        impunge_sh->entrybuf = impunge_sh->buf[active_src];
-        impunge_sh->parentbuf = impunge_sh->parentbufs[active_src];
-        recreate_count = afr_sh_recreate_count (impunge_sh, sh->sources,
-                                                priv->child_count);
-        if (!recreate_count) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this, 0, 0);
-                goto out;
-        }
-        impunge_local->call_count = recreate_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!impunge_local->child_up[i]) {
-                        impunge_sh->child_errno[i] = ENOTCONN;
-                        continue;
-                }
-                if (!afr_sh_need_recreate (impunge_sh, i, priv->child_count)) {
-                        impunge_sh->child_errno[i] = EEXIST;
-                        continue;
-                }
-        }
-        for (i = 0; i < priv->child_count; i++) {
-                if (!afr_sh_need_recreate (impunge_sh, i, priv->child_count))
-                        continue;
-                (void)afr_sh_entry_impunge_create (impunge_frame, this, i);
-                recreate_count--;
-        }
-        GF_ASSERT (!recreate_count);
-out:
-        return 0;
-}
 
-void
-afr_sh_entry_common_lookup_done (call_frame_t *impunge_frame, xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        unsigned int     gfid_miss_count = 0;
-        unsigned int     children_up_count = 0;
-        uuid_t           gfid = {0};
-        int              active_src = 0;
-
-        priv          = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src    = impunge_sh->active_source;
-
-        if (op_ret < 0)
-                goto done;
-        if (impunge_sh->child_errno[active_src]) {
-                op_ret = -1;
-                op_errno = impunge_sh->child_errno[active_src];
-                goto done;
-        }
-
-        gfid_miss_count = afr_gfid_missing_count (this->name,
-                                                  impunge_sh->success_children,
-                                                  impunge_sh->buf, priv->child_count,
-                                                  impunge_local->loc.path);
-        children_up_count = afr_up_children_count (impunge_local->child_up,
-                                                   priv->child_count);
-        if ((gfid_miss_count == children_up_count) &&
-            (children_up_count < priv->child_count)) {
-                op_ret = -1;
-                op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "Not all children are up, "
-                        "gfid should not be assigned in this state for %s",
-                        impunge_local->loc.path);
-                goto done;
-        }
-
-        if (gfid_miss_count) {
-                afr_update_gfid_from_iatts (gfid, impunge_sh->buf,
-                                            impunge_sh->success_children,
-                                            priv->child_count);
-                if (uuid_is_null (gfid)) {
-                        sh->entries_skipped = _gf_true;
-                        gf_log (this->name, GF_LOG_INFO, "%s: Skipping entry "
-                                "self-heal because of gfid absence",
-                                impunge_local->loc.path);
-                        goto done;
-                }
-                afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
-                                      afr_sh_entry_common_lookup_done, gfid,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
-        } else {
-                afr_sh_entry_call_impunge_recreate (impunge_frame, this);
-        }
-        return;
-done:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        op_ret, op_errno);
-        return;
-}
-
-int
-afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
-                            gf_dirent_t *entry)
-{
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        afr_self_heal_t *impunge_sh  = NULL;
-        int              ret = -1;
-        call_frame_t    *impunge_frame = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              active_src = 0;
-        int              op_errno = 0;
-        int              op_ret = -1;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-        sh->impunge_done = afr_sh_entry_impunge_entry_done;
-
-        if ((strcmp (entry->d_name, ".") == 0)
-            || (strcmp (entry->d_name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        entry->d_name, local->loc.path);
-                op_ret = 0;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "inspecting existence of %s under %s",
-                entry->d_name, local->loc.path);
-
-        ret = afr_impunge_frame_create (frame, this, active_src,
-                                        &impunge_frame);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        ret = afr_build_child_loc (this, &impunge_local->loc, &local->loc,
-                                   entry->d_name);
-        loc_copy (&impunge_sh->parent_loc, &local->loc);
-        if (ret != 0) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
-                              afr_sh_entry_common_lookup_done, NULL,
-                              AFR_LOOKUP_FAIL_CONFLICTS);
-
-        op_ret = 0;
+static int
+__afr_selfheal_entry (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      unsigned char *locked_on)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
+
+	priv = this->private;
+
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+
+	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
+				    data_lock);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_entry_prepare (frame, this, fd, data_lock,
+						    sources, sinks, healed_sinks,
+						    locked_replies, &source);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
+				data_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = afr_selfheal_entry_do (frame, this, fd, source, sources,
+				     healed_sinks, locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, fd->inode, sources, sinks,
+					 healed_sinks, AFR_ENTRY_TRANSACTION,
+					 locked_replies, data_lock);
 out:
-        if (ret) {
-                if (impunge_frame)
-                        AFR_STACK_DESTROY (impunge_frame);
-                sh->impunge_done (frame, this, op_ret, op_errno);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        gf_dirent_t     *entry = NULL;
-        off_t            last_offset = 0;
-        int              active_src = 0;
-        int              entry_count = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "readdir of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[active_src]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "readdir of %s on subvolume %s complete",
-                                local->loc.path,
-                                priv->children[active_src]->name);
-                }
-
-                afr_sh_entry_impunge_all (frame, this);
-                return 0;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-                entry_count++;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "readdir'ed %d entries from %s",
-                entry_count, priv->children[active_src]->name);
-
-        sh->offset = last_offset;
-        local->call_count = entry_count;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                afr_sh_entry_impunge_entry (frame, this, entry);
-        }
-
-        return 0;
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this)
+static fd_t *
+afr_selfheal_data_opendir (xlator_t *this, inode_t *inode)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int32_t         active_src = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-        active_src = sh->active_source;
-        gf_log (this->name, GF_LOG_DEBUG, "%s: readdir from offset %zd",
-                local->loc.path, sh->offset);
-
-        STACK_WIND (frame, afr_sh_entry_impunge_readdir_cbk,
-                    priv->children[active_src],
-                    priv->children[active_src]->fops->readdirp,
-                    sh->healing_fd, sh->block_size, sh->offset, 0);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              active_src = -1;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
+	loc_t loc = {0,};
+	int ret = 0;
+	fd_t *fd = NULL;
 
-        sh->offset = 0;
+	fd = fd_create (inode, 0);
+	if (!fd)
+		return NULL;
 
-        active_src = next_active_source (frame, this, sh->active_source);
-        sh->active_source = active_src;
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        if (sh->op_failed) {
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
+	ret = syncop_opendir (this, &loc, fd);
+	if (ret) {
+		fd_unref (fd);
+		fd = NULL;
+	} else {
+		fd_bind (fd);
+	}
 
-        if (active_src == -1) {
-                /* completed creating missing files on all subvolumes */
-                afr_sh_entry_erase_pending (frame, this);
-                return 0;
-        }
+	loc_wipe (&loc);
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "impunging entries of %s on %s to other sinks",
-                local->loc.path, priv->children[active_src]->name);
-
-        afr_sh_entry_impunge_subvol (frame, this);
-
-        return 0;
+	return fd;
 }
 
 
-int
-afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        /* TODO: some of the open's might fail.
-           In that case, modify cleanup fn to send flush on those
-           fd's which are already open */
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "opendir of %s failed on child %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->op_failed) {
-                        afr_sh_entry_finish (frame, this);
-                        return 0;
-                }
-                gf_log (this->name, GF_LOG_TRACE,
-                        "fd for %s opened, commencing sync",
-                        local->loc.path);
-
-                sh->active_source = -1;
-                afr_sh_entry_expunge_all (frame, this);
-        }
-
-        return 0;
-}
-
 
 int
-afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
+afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        int i = 0;
-        int call_count = 0;
-
-        int source = -1;
-        int *sources = NULL;
-
-        fd_t *fd = NULL;
-
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source  = local->self_heal.source;
-        sources = local->self_heal.sources;
-
-        sh->block_size = 65536; //131072
-        sh->offset = 0;
-
-        call_count = sh->active_sinks;
-        if (source != -1)
-                call_count++;
-
-        local->call_count = call_count;
-
-        fd = fd_create (local->loc.inode, frame->root->pid);
-        sh->healing_fd = fd;
-
-        if (source != -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "opening directory %s on subvolume %s (source)",
-                        local->loc.path, priv->children[source]->name);
-
-                /* open source */
-                STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-                                   (void *) (long) source,
-                                   priv->children[source],
-                                   priv->children[source]->fops->opendir,
-                                   &local->loc, fd);
-                call_count--;
-        }
-
-        /* open sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] || !local->child_up[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "opening directory %s on subvolume %s (sink)",
-                        local->loc.path, priv->children[i]->name);
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	fd_t *fd = NULL;
+	int ret = 0;
 
-                STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->opendir,
-                                   &local->loc, fd);
+	priv = this->private;
 
-                if (!--call_count)
-                        break;
-        }
+	fd = afr_selfheal_data_opendir (this, inode);
+	if (!fd)
+		return -EIO;
 
-        return 0;
-}
-
-
-int
-afr_sh_entry_sync_prepare (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        afr_sh_mark_source_sinks (frame, this);
-        if (source != -1)
-                sh->success[source] = 1;
-
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "no active sinks for self-heal on dir %s",
-                        local->loc.path);
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
-        if (source == -1 && sh->active_sinks < 2) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "cannot sync with 0 sources and 1 sink on dir %s",
-                        local->loc.path);
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
-
-        if (source != -1)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "self-healing directory %s from subvolume %s to "
-                        "%d other",
-                        local->loc.path, priv->children[source]->name,
-                        sh->active_sinks);
-        else
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sources for %s found. "
-                        "merging all entries as a conservative decision",
-                        local->loc.path);
-
-        afr_sh_entry_open (frame, this);
-
-        return 0;
-}
-
-
-void
-afr_sh_entry_fix (call_frame_t *frame, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-        int              nsources = 0;
-        int32_t          subvol_status = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                afr_sh_entry_finish (frame, this);
-                goto out;
-        }
-
-        if (sh->forced_merge) {
-                sh->source = -1;
-                goto heal;
-        }
-
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_ENTRY_TRANSACTION, &subvol_status,
-                                      _gf_true);
-        if ((subvol_status & ALL_FOOLS) ||
-            (subvol_status & SPLIT_BRAIN)) {
-                gf_log (this->name, GF_LOG_INFO, "%s: Performing conservative "
-                        "merge", local->loc.path);
-                source = -1;
-                memset (sh->sources, 0,
-                        sizeof (*sh->sources) * priv->child_count);
-        } else if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_entry_finish (frame, this);
-                return;
-        } else {
-                source = afr_sh_select_source (sh->sources, priv->child_count);
-        }
-
-        sh->source = source;
-
-        afr_reset_children (sh->fresh_children, priv->child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        if (sh->source >= 0)
-                afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                        sh->fresh_children);
-
-heal:
-        afr_sh_entry_sync_prepare (frame, this);
-out:
-        return;
-}
-
-int
-afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Non Blocking entrylks "
-                        "failed for %s.", local->loc.path);
-                sh->op_failed = 1;
-                afr_sh_entry_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking entrylks done "
-                        "for %s. Proceeding to FOP", local->loc.path);
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_entry_fix, NULL,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
-        }
-
-        return 0;
-}
-
-int
-afr_self_heal_entry (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t   *priv = NULL;
+	locked_on = alloca0 (priv->child_count);
 
+	ret = afr_selfheal_tryentrylk (frame, this, inode, priv->sh_domain, NULL,
+				       locked_on);
+	{
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
+		}
 
-        priv = this->private;
-        local = frame->local;
+		ret = __afr_selfheal_entry (frame, this, fd, locked_on);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain, NULL, locked_on);
 
-        if (local->self_heal.do_entry_self_heal && priv->entry_self_heal) {
-                afr_sh_entrylk (frame, this, &local->loc, NULL,
-                                afr_sh_post_nonblocking_entry_cbk);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to completion on %s",
-                        local->loc.path);
-                afr_sh_entry_done (frame, this);
-        }
+	if (fd)
+		fd_unref (fd);
 
-        return 0;
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index d0bf382a4..b31a33237 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -1,606 +1,281 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-
-int
-afr_sh_metadata_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        afr_sh_reset (frame, this);
-        if (local->govinda_gOvinda) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "split-brain detected, aborting selfheal of %s",
-                        local->loc.path);
-                sh->op_failed = 1;
-                sh->completion_cbk (frame, this);
-        } else {
-                if (IA_ISDIR (sh->type)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "proceeding to entry check on %s",
-                                local->loc.path);
-                        afr_self_heal_entry (frame, this);
-                        return 0;
-                }
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "proceeding to data check on %s",
-                        local->loc.path);
-                afr_self_heal_data (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_inode_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_metadata_done;
-        afr_unlock (frame, this);
-
-        return 0;
-}
-
-int
-afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
-{
-        afr_sh_inode_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, dict_t *xattr)
-{
-        afr_local_t     *local     = NULL;
-        int             call_count = 0;
-        long            i          = 0;
-        afr_self_heal_t *sh        = NULL;
-        afr_private_t   *priv      = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-        sh = &local->self_heal;
-        i = (long)cookie;
-
-        if ((!IA_ISREG (sh->buf[sh->source].ia_type)) &&
-            (!IA_ISDIR (sh->buf[sh->source].ia_type))) {
-                afr_children_add_child (sh->fresh_children, i,
-                                        priv->child_count);
-        }
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if ((!IA_ISREG (sh->buf[sh->source].ia_type)) &&
-                    (!IA_ISDIR (sh->buf[sh->source].ia_type))) {
-                        afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                                sh->fresh_children);
-                }
-                afr_sh_metadata_finish (frame, this);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix,
-                                 sh->success, priv->child_count,
-                                 AFR_METADATA_TRANSACTION);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-        if (!erase_xattr)
-                return -ENOMEM;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_METADATA_TRANSACTION);
-
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "metadata of %s not healed on any subvolume",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &local->loc,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "setting attributes failed for %s on %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-
-                        sh->success[child_index] = 0;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                afr_sh_metadata_erase_pending (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *preop, struct iatt *postop)
-{
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
-{
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
+#include "byte-order.h"
 
-        return 0;
-}
 
+#define AFR_HEAL_ATTR (GF_SET_ATTR_UID|GF_SET_ATTR_GID|GF_SET_ATTR_MODE)
 
 int
-afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
+afr_selfheal_metadata_do (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			  int source, unsigned char *healed_sinks,
+			  struct afr_reply *locked_replies)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-        int              active_sinks = 0;
-        int              call_count = 0;
-        int              i = 0;
-
-        struct iatt      stbuf = {0,};
-        int32_t          valid = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-        active_sinks = sh->active_sinks;
-
-        /*
-         * 2 calls per sink - setattr, setxattr
-         */
-        if (xattr)
-                call_count = active_sinks * 2;
-        else
-                call_count = active_sinks;
-
-        local->call_count = call_count;
-
-        stbuf.ia_atime = sh->buf[source].ia_atime;
-        stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-        stbuf.ia_mtime = sh->buf[source].ia_mtime;
-        stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
-
-        stbuf.ia_uid = sh->buf[source].ia_uid;
-        stbuf.ia_gid = sh->buf[source].ia_gid;
-
-        stbuf.ia_type = sh->buf[source].ia_type;
-        stbuf.ia_prot = sh->buf[source].ia_prot;
-
-        valid = GF_SET_ATTR_MODE  |
-                GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (call_count == 0) {
-                        break;
-                }
-                if (sh->sources[i] || !local->child_up[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "self-healing metadata of %s from %s to %s",
-                        local->loc.path, priv->children[source]->name,
-                        priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_setattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid);
-
-                call_count--;
-
-                if (!xattr)
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setxattr,
-                                   &local->loc, xattr, 0);
-                call_count--;
-        }
-
-        return 0;
+	int ret = -1;
+	loc_t loc = {0,};
+	dict_t *xattr = NULL;
+	dict_t *old_xattr = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	priv = this->private;
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+
+	gf_log (this->name, GF_LOG_INFO, "performing metadata selfheal on %s",
+		uuid_utoa (inode->gfid));
+
+	ret = syncop_getxattr (priv->children[source], &loc, &xattr, NULL);
+	if (ret < 0) {
+		loc_wipe (&loc);
+		return -EIO;
+	}
+
+	afr_filter_xattrs (xattr);
+	dict_del (xattr, GF_SELINUX_XATTR_KEY);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+
+		ret = syncop_setattr (priv->children[i], &loc,
+				      &locked_replies[source].poststat,
+				      AFR_HEAL_ATTR, NULL, NULL);
+		if (ret)
+			healed_sinks[i] = 0;
+
+		old_xattr = NULL;
+		ret = syncop_getxattr (priv->children[i], &loc, &old_xattr, 0);
+		if (old_xattr) {
+			dict_del (old_xattr, GF_SELINUX_XATTR_KEY);
+			afr_filter_xattrs (old_xattr);
+			ret = syncop_removexattr (priv->children[i], &loc, "",
+						  old_xattr);
+		}
+
+		ret = syncop_setxattr (priv->children[i], &loc, xattr, 0);
+		if (ret)
+			healed_sinks[i] = 0;
+	}
+
+	loc_wipe (&loc);
+	if (xattr)
+		dict_unref (xattr);
+
+	return 0;
 }
 
 
-int
-afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie,
-                              xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xattr)
+/*
+ * Look for mismatching uid/gid or mode even if xattrs don't say so, and
+ * pick one arbitrarily as winner.
+ */
+
+static int
+__afr_selfheal_metadata_finalize_source (xlator_t *this, unsigned char *sources,
+					 unsigned char *sinks,
+					 unsigned char *locked_on,
+					 struct afr_reply *replies)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        int i;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "getxattr of %s failed on subvolume %s (%s). proceeding without xattr",
-                        local->loc.path, priv->children[source]->name,
-                        strerror (op_errno));
-
-                afr_sh_metadata_sync (frame, this, NULL);
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        dict_del (xattr, priv->pending_key[i]);
-                }
-
-                afr_sh_metadata_sync (frame, this, xattr);
-        }
-
-        return 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	struct iatt first = {0, };
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
+
+	priv = this->private;
+
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
+
+	if (locked_count == sinks_count || !sources_count) {
+		if (!priv->metadata_splitbrain_forced_heal) {
+			return -EIO;
+		}
+		/* Metadata split brain, select one subvol
+		   arbitrarily */
+		for (i = 0; i < priv->child_count; i++) {
+			if (locked_on[i] && sinks[i]) {
+				sources[i] = 1;
+				sinks[i] = 0;
+				break;
+			}
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (source == -1) {
+			source = i;
+			first = replies[i].poststat;
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (!IA_EQUAL (first, replies[i].poststat, type) ||
+		    !IA_EQUAL (first, replies[i].poststat, uid) ||
+		    !IA_EQUAL (first, replies[i].poststat, gid) ||
+		    !IA_EQUAL (first, replies[i].poststat, prot)) {
+			sources[i] = 0;
+			sinks[i] = 1;
+		}
+	}
+
+	return source;
 }
 
 
-int
-afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
+static int
+__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this, inode_t *inode,
+				 unsigned char *locked_on, unsigned char *sources,
+				 unsigned char *sinks, unsigned char *healed_sinks,
+				 struct afr_reply *replies)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        afr_sh_mark_source_sinks (frame, this);
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sinks for performing self-heal on file %s",
-                        local->loc.path);
-                afr_sh_metadata_finish (frame, this);
-                return 0;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "syncing metadata of %s from subvolume %s to %d active sinks",
-                local->loc.path, priv->children[source]->name,
-                sh->active_sinks);
-
-        STACK_WIND (frame, afr_sh_metadata_getxattr_cbk,
-                    priv->children[source],
-                    priv->children[source]->fops->getxattr,
-                    &local->loc, NULL);
-
-        return 0;
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	priv = this->private;
+
+	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
+
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_METADATA_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
+
+	source = __afr_selfheal_metadata_finalize_source (this, sources, sinks,
+							  locked_on, replies);
+	if (source < 0)
+		return -EIO;
+
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
+
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
+
+	return source;
 }
 
 
-void
-afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+static int
+__afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              nsources = 0;
-        int              source = 0;
-        int              i = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_METADATA_TRANSACTION, NULL, _gf_false);
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-
-        if ((nsources == -1)
-            && (priv->favorite_child != -1)
-            && (sh->child_errno[priv->favorite_child] == 0)) {
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Picking favorite child %s as authentic source to resolve conflicting metadata of %s",
-                        priv->children[priv->favorite_child]->name,
-                        local->loc.path);
-
-                sh->sources[priv->favorite_child] = 1;
-
-                nsources = afr_sh_source_count (sh->sources,
-                                                priv->child_count);
-        }
-
-        if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal permissions/ownership of '%s' "
-                        "(possible split-brain). Please fix the file on "
-                        "all backend volumes", local->loc.path);
-
-                local->govinda_gOvinda = 1;
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-
-        source = afr_sh_select_source (sh->sources, priv->child_count);
-
-        if (source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No active sources found.");
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-
-        sh->source = source;
-
-        /* detect changes not visible through pending flags -- JIC */
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == source || sh->child_errno[i])
-                        continue;
-
-                if (PERMISSION_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
-
-                if (OWNERSHIP_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
-        }
-
-        if ((!IA_ISREG (sh->buf[source].ia_type)) &&
-            (!IA_ISDIR (sh->buf[source].ia_type))) {
-                afr_reset_children (sh->fresh_children, priv->child_count);
-                afr_get_fresh_children (sh->success_children, sh->sources,
-                                        sh->fresh_children, priv->child_count);
-                afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                        sh->fresh_children);
-        }
-
-        afr_sh_metadata_sync_prepare (frame, this);
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
+
+	priv = this->private;
+
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+
+	ret = afr_selfheal_inodelk (frame, this, inode, this->name,
+				    LLONG_MAX - 1, 0, data_lock);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_metadata_prepare (frame, this, inode, data_lock,
+						       sources, sinks, healed_sinks,
+						       locked_replies);
+		if (ret < 0)
+			goto unlock;
+
+		source = ret;
+		ret = 0;
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, this->name,
+				LLONG_MAX -1, 0, data_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = afr_selfheal_metadata_do (frame, this, inode, source, healed_sinks,
+					locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, inode, sources, sinks,
+					 healed_sinks, AFR_METADATA_TRANSACTION,
+					 locked_replies, data_lock);
 out:
-        return;
+	return ret;
 }
 
-int
-afr_sh_metadata_post_nonblocking_inodelk_cbk (call_frame_t *frame,
-                                              xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Non Blocking metadata "
-                        "inodelks failed for %s.", local->loc.path);
-                gf_log (this->name, GF_LOG_ERROR, "Metadata self-heal "
-                        "failed for %s.", local->loc.path);
-                afr_sh_metadata_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking metadata "
-                        "inodelks done for %s. Proceeding to FOP",
-                        local->loc.path);
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_metadata_fix, NULL,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS);
-        }
-
-        return 0;
-}
 
 int
-afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
+afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_METADATA_SELF_HEAL_LK;
-
-        afr_set_lock_number (frame, this);
-
-        int_lock->lk_flock.l_start = LLONG_MAX - 1;
-        int_lock->lk_flock.l_len   = 0;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-        int_lock->lock_cbk         = afr_sh_metadata_post_nonblocking_inodelk_cbk;
-
-        afr_nonblocking_inodelk (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv = this->private;
-
-
-        local = frame->local;
-
-        if (local->self_heal.do_metadata_self_heal && priv->metadata_self_heal) {
-                afr_sh_metadata_lock (frame, this);
-        } else {
-                afr_sh_metadata_done (frame, this);
-        }
-
-        return 0;
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	int ret = 0;
+
+	priv = this->private;
+
+	locked_on = alloca0 (priv->child_count);
+
+	ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain, 0, 0,
+				       locked_on);
+	{
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_metadata (frame, this, inode, locked_on);
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0, locked_on);
+
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c
new file mode 100644
index 000000000..ce80b8da3
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heal-name.c
@@ -0,0 +1,457 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "afr.h"
+#include "afr-self-heal.h"
+
+
+int
+__afr_selfheal_assign_gfid (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			    uuid_t pargfid, const char *bname, inode_t *inode,
+			    struct afr_reply *replies, int gfid_idx)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	dict_t *xdata = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+
+	priv = this->private;
+
+	uuid_copy (parent->gfid, pargfid);
+
+	xdata = dict_new ();
+	if (!xdata) {
+		return -ENOMEM;
+	}
+
+	ret = dict_set_static_bin (xdata, "gfid-req",
+				   replies[gfid_idx].poststat.ia_gfid, 16);
+	if (ret) {
+		dict_destroy (xdata);
+		return -ENOMEM;
+	}
+
+	loc.parent = inode_ref (parent);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.pargfid, pargfid);
+	loc.name = bname;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (replies[i].op_ret == 0 || replies[i].op_errno != ENODATA)
+			continue;
+
+		ret = syncop_lookup (priv->children[i], &loc, xdata, 0, 0, 0);
+	}
+
+	loc_wipe (&loc);
+	dict_unref (xdata);
+
+	return ret;
+}
+
+
+int
+__afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, const char *bname, inode_t *inode,
+			     struct afr_reply *replies, int gfid_idx)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int ret = 0;
+
+	priv = this->private;
+
+	uuid_copy (parent->gfid, pargfid);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[gfid_idx].poststat.ia_gfid) == 0)
+			continue;
+
+		ret |= afr_selfheal_recreate_entry (frame, this, i, gfid_idx,
+						    parent, bname, inode, replies);
+	}
+
+	return ret;
+}
+
+
+int
+__afr_selfheal_name_expunge (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, const char *bname, inode_t *inode,
+			     struct afr_reply *replies)
+{
+	loc_t loc = {0, };
+	int i = 0;
+	afr_private_t *priv = NULL;
+	char g[64];
+	int ret = 0;
+
+	priv = this->private;
+
+	loc.parent = inode_ref (parent);
+	uuid_copy (loc.pargfid, pargfid);
+	loc.name = bname;
+	loc.inode = inode_ref (inode);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (replies[i].op_ret)
+			continue;
+
+		switch (replies[i].poststat.ia_type) {
+		case IA_IFDIR:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging dir %s/%s (%s) on %s",
+				uuid_utoa (pargfid), bname,
+				uuid_utoa_r (replies[i].poststat.ia_gfid, g),
+				priv->children[i]->name);
+			ret |= syncop_rmdir (priv->children[i], &loc, 1);
+			break;
+		default:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging file %s/%s (%s) on %s",
+				uuid_utoa (pargfid), bname,
+				uuid_utoa_r (replies[i].poststat.ia_gfid, g),
+				priv->children[i]->name);
+			ret |= syncop_unlink (priv->children[i], &loc);
+			break;
+		}
+	}
+
+	loc_wipe (&loc);
+
+	return ret;
+
+}
+
+
+int
+__afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			uuid_t pargfid, const char *bname, inode_t *inode,
+			unsigned char *sources, unsigned char *sinks,
+			unsigned char *healed_sinks, int source,
+			unsigned char *locked_on, struct afr_reply *replies)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	uuid_t gfid = {0, };
+	int gfid_idx = -1;
+	gf_boolean_t source_is_empty = _gf_true;
+	gf_boolean_t need_heal = _gf_false;
+	int first_idx = -1;
+	char g1[64],g2[64];
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first_idx == -1) {
+			first_idx = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first_idx].op_ret)
+			need_heal = _gf_true;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first_idx].poststat.ia_gfid))
+			need_heal = _gf_true;
+	}
+
+	if (!need_heal)
+		return 0;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (!replies[i].op_ret && (source == -1 || sources[i])) {
+			source_is_empty = _gf_false;
+			break;
+		}
+	}
+
+	if (source_is_empty) {
+		return __afr_selfheal_name_expunge (frame, this, parent, pargfid,
+						    bname, inode, replies);
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (uuid_is_null (replies[i].poststat.ia_gfid))
+			continue;
+
+		if (uuid_is_null (gfid)) {
+			uuid_copy (gfid, replies[i].poststat.ia_gfid);
+			gfid_idx = i;
+			continue;
+		}
+
+		if (sources[i] || source == -1) {
+			if (gfid_idx != -1 &&
+			    (sources[gfid_idx] || source == -1) &&
+			    uuid_compare (gfid, replies[i].poststat.ia_gfid)) {
+				gf_log (this->name, GF_LOG_WARNING,
+					"GFID mismatch for <gfid:%s>/%s "
+					"%s on %s and %s on %s",
+					uuid_utoa (pargfid), bname,
+					uuid_utoa_r (replies[i].poststat.ia_gfid, g1),
+					priv->children[i]->name,
+					uuid_utoa_r (replies[gfid_idx].poststat.ia_gfid, g2),
+					priv->children[gfid_idx]->name);
+				return -1;
+			}
+
+			uuid_copy (gfid, replies[i].poststat.ia_gfid);
+			gfid_idx = i;
+			continue;
+		}
+	}
+
+	if (gfid_idx == -1)
+		return -1;
+
+	__afr_selfheal_assign_gfid (frame, this, parent, pargfid, bname, inode,
+				    replies, gfid_idx);
+
+	return __afr_selfheal_name_impunge (frame, this, parent, pargfid,
+					    bname, inode, replies, gfid_idx);
+}
+
+
+int
+__afr_selfheal_name_finalize_source (xlator_t *this, unsigned char *sources,
+				     unsigned char *sinks, unsigned char *locked_on,
+				     struct afr_reply *replies)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
+
+	priv = this->private;
+
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
+
+	if (locked_count == sinks_count || !sources_count) {
+		return -1;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i]) {
+			source = i;
+			break;
+		}
+	}
+
+	return source;
+}
+
+
+int
+__afr_selfheal_name_prepare (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks,
+			     unsigned char *healed_sinks, struct afr_reply *replies,
+			     int *source_p)
+{
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	priv = this->private;
+
+	ret = afr_selfheal_unlocked_discover (frame, parent, pargfid, replies);
+	if (ret)
+		return ret;
+
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_ENTRY_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
+
+	source = __afr_selfheal_name_finalize_source (this, sources, sinks,
+						      locked_on, replies);
+	if (source < 0) {
+		/* If source is < 0 (typically split-brain), we perform a
+		   conservative merge of entries rather than erroring out */
+	}
+	*source_p = source;
+
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
+
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
+
+	return ret;
+}
+
+
+int
+afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
+		      uuid_t pargfid, const char *bname)
+{
+	afr_private_t *priv = NULL;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *healed_sinks = NULL;
+	unsigned char *locked_on = NULL;
+	int source = -1;
+	struct afr_reply *replies = NULL;
+	int ret = -1;
+	inode_t *inode = NULL;
+
+	priv = this->private;
+
+	locked_on = alloca0 (priv->child_count);
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+
+	replies = alloca0 (priv->child_count * sizeof(*replies));
+
+	ret = afr_selfheal_entrylk (frame, this, parent, this->name, bname,
+				    locked_on);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_name_prepare (frame, this, parent, pargfid,
+						   locked_on, sources, sinks,
+						   healed_sinks, replies,
+						   &source);
+		if (ret)
+			goto unlock;
+
+		inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
+							 replies, locked_on);
+		if (!inode) {
+			ret = -ENOMEM;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_name_do (frame, this, parent, pargfid, bname,
+					      inode, sources, sinks, healed_sinks,
+					      source, locked_on, replies);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, parent, this->name, bname,
+				locked_on);
+	if (inode)
+		inode_unref (inode);
+
+	return ret;
+}
+
+
+int
+afr_selfheal_name_unlocked_inspect (call_frame_t *frame, xlator_t *this,
+				    inode_t *parent, uuid_t pargfid,
+				    const char *bname, gf_boolean_t *need_heal)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	struct afr_reply *replies = NULL;
+	inode_t *inode = NULL;
+	int first_idx = -1;
+
+	priv = this->private;
+
+	replies = alloca0 (sizeof (*replies) * priv->child_count);
+
+	inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
+						 replies, priv->child_up);
+	if (!inode)
+		return -ENOMEM;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first_idx == -1) {
+			first_idx = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first_idx].op_ret)
+			*need_heal = _gf_true;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first_idx].poststat.ia_gfid))
+			*need_heal = _gf_true;
+	}
+
+	if (inode)
+		inode_unref (inode);
+	return 0;
+}
+
+int
+afr_selfheal_name (xlator_t *this, uuid_t pargfid, const char *bname)
+{
+	inode_t *parent = NULL;
+	call_frame_t *frame = NULL;
+	int ret = -1;
+	gf_boolean_t need_heal = _gf_false;
+
+	parent = afr_inode_find (this, pargfid);
+	if (!parent)
+		goto out;
+
+	frame = afr_frame_create (this);
+	if (!frame)
+		goto out;
+
+	ret = afr_selfheal_name_unlocked_inspect (frame, this, parent, pargfid,
+						  bname, &need_heal);
+	if (ret)
+		goto out;
+
+	if (need_heal)
+		afr_selfheal_name_do (frame, this, parent, pargfid, bname);
+out:
+	if (parent)
+		inode_unref (parent);
+	if (frame)
+		AFR_STACK_DESTROY (frame);
+
+	return ret;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index f40c06faa..a1b972ac3 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -1,58 +1,167 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef __AFR_SELF_HEAL_H__
-#define __AFR_SELF_HEAL_H__
 
-#include <sys/stat.h>
+#ifndef _AFR_SELFHEAL_H
+#define _AFR_SELFHEAL_H
+
+
+/* Perform fop on all UP subvolumes and wait for all callbacks to return */
+
+#define AFR_ONALL(frame, rfn, fop, args ...) do {			\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0, __count = 0;					\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!__priv->child_up[__i]) continue;			\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		__count++;						\
+	}								\
+	syncbarrier_wait (&__local->barrier, __count);			\
+	} while (0)
+
+
+/* Perform fop on all subvolumes represented by list[] array and wait
+   for all callbacks to return */
+
+#define AFR_ONLIST(list, frame, rfn, fop, args ...) do {		\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0, __count = 0;					\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!list[__i]) continue;				\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		__count++;						\
+	}								\
+	syncbarrier_wait (&__local->barrier, __count);			\
+	} while (0)
+
+
+#define AFR_SEQ(frame, rfn, fop, args ...) do {				\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0;							\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!__priv->child_up[__i]) continue;			\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		syncbarrier_wait (&__local->barrier, 1);		\
+	}								\
+	} while (0)
 
-#define FILETYPE_DIFFERS(buf1,buf2) ((buf1)->ia_type != (buf2)->ia_type)
-#define PERMISSION_DIFFERS(buf1,buf2) (st_mode_from_ia ((buf1)->ia_prot, (buf1)->ia_type) != st_mode_from_ia ((buf2)->ia_prot, (buf2)->ia_type))
-#define OWNERSHIP_DIFFERS(buf1,buf2) (((buf1)->ia_uid != (buf2)->ia_uid) || ((buf1)->ia_gid != (buf2)->ia_gid))
-#define SIZE_DIFFERS(buf1,buf2) ((buf1)->ia_size != (buf2)->ia_size)
 
-#define SIZE_GREATER(buf1,buf2) ((buf1)->ia_size > (buf2)->ia_size)
+#define ALLOC_MATRIX(n, type) ({type **__ptr = NULL; \
+	int __i; \
+	__ptr = alloca0 (n * sizeof(type *)); \
+	for (__i = 0; __i < n; __i++) __ptr[__i] = alloca0 (n * sizeof(type)); \
+	__ptr;})
+
+
+#define IA_EQUAL(f,s,field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0)
+
 
 int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this);
+afr_selfheal (xlator_t *this, uuid_t gfid);
+
 int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this);
+afr_selfheal_name (xlator_t *this, uuid_t gfid, const char *name);
+
 int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this);
+afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-afr_self_heal_entry (call_frame_t *frame, xlator_t *this);
+afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this);
+afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+
+int
+afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, off_t off, size_t size,
+		      unsigned char *locked_on);
 
 int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this);
+afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, off_t off, size_t size,
+			 unsigned char *locked_on);
 
 int
-afr_self_heal_get_source (xlator_t *this, afr_local_t *local, dict_t **xattr);
+afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, off_t off, size_t size,
+			const unsigned char *locked_on);
 
 int
-afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
-                                          dict_t **xattr,
-                                          afr_transaction_type txn_type);
-#endif /* __AFR_SELF_HEAL_H__ */
+afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, const char *name, unsigned char *locked_on);
+
+int
+afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, const char *name, unsigned char *locked_on);
+
+int
+afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
+				uuid_t gfid, struct afr_reply *replies);
+
+inode_t *
+afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
+				 const char *name, struct afr_reply *replies,
+				 unsigned char *lookup_on);
+
+int
+afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
+			     struct afr_reply *replies,
+			     afr_transaction_type type, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks);
+
+int
+afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
+			    afr_transaction_type type, int *dirty, int **matrix);
+
+int
+afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			   unsigned char *sources, unsigned char *sinks,
+			   unsigned char *healed_sinks, afr_transaction_type type,
+			   struct afr_reply *replies, unsigned char *locked_on);
+
+int
+afr_selfheal_recreate_entry (call_frame_t *frame, xlator_t *this, int dst,
+			     int source, inode_t *dir, const char *name,
+			     inode_t *inode, struct afr_reply *replies);
+
+int
+afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      int subvol, dict_t *xattr);
+
+call_frame_t *
+afr_frame_create (xlator_t *this);
+
+inode_t *
+afr_inode_find (xlator_t *this, uuid_t gfid);
+
+#endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 36d2cfb6e..4bfe909bc 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -1,1088 +1,1256 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
+
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
+
 #include "afr.h"
-#include "syncop.h"
+#include "afr-self-heal.h"
 #include "afr-self-heald.h"
-#include "afr-self-heal-common.h"
 #include "protocol-common.h"
-#include "event-history.h"
 
-#define AFR_POLL_TIMEOUT 600
+#define SHD_INODE_LRU_LIMIT          2048
+#define AFR_EH_HEALED_LIMIT          1024
+#define AFR_EH_HEAL_FAIL_LIMIT       1024
+#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
+#define AFR_STATISTICS_HISTORY_SIZE    50
 
-typedef enum {
-        STOP_CRAWL_ON_SINGLE_SUBVOL = 1
-} afr_crawl_flags_t;
 
-typedef struct shd_dump {
-        dict_t   *dict;
-        time_t   sh_time;
-        xlator_t *this;
-        int      child;
-} shd_dump_t;
+#define ASSERT_LOCAL(this, healer)				\
+	if (!afr_shd_is_subvol_local(this, healer->subvol)) {	\
+		healer->local = _gf_false;			\
+		if (safe_break (healer)) {			\
+			break;					\
+		} else {					\
+			continue;				\
+		}						\
+	} else {						\
+		healer->local = _gf_true;			\
+	}
 
-typedef struct shd_event_ {
-        int     child;
-        char    *path;
-} shd_event_t;
 
-typedef int
-(*afr_crawl_done_cbk_t)  (int ret, call_frame_t *sync_frame, void *crawl_data);
+#define NTH_INDEX_HEALER(this, n) &((((afr_private_t *)this->private))->shd.index_healers[n])
+#define NTH_FULL_HEALER(this, n) &((((afr_private_t *)this->private))->shd.full_healers[n])
 
-void
-afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
-                 process_entry_cbk_t process_entry, void *op_data,
-                 gf_boolean_t exclusive, int crawl_flags,
-                 afr_crawl_done_cbk_t crawl_done);
+int afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid, char **path_p);
 
-static int
-_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data);
-
-void
-shd_cleanup_event (void *event)
+char *
+afr_subvol_name (xlator_t *this, int subvol)
 {
-        shd_event_t *shd_event = event;
+	afr_private_t *priv = NULL;
 
-        if (!shd_event)
-                goto out;
-        if (shd_event->path)
-                GF_FREE (shd_event->path);
-        GF_FREE (shd_event);
-out:
-        return;
+	priv = this->private;
+	if (subvol < 0 || subvol > priv->child_count)
+		return NULL;
+
+	return priv->children[subvol]->name;
 }
 
-int
-afr_get_local_child (afr_self_heald_t *shd, unsigned int child_count)
+
+void
+afr_destroy_crawl_event_data (void *data)
 {
-        int i = 0;
-        int ret = -1;
-        for (i = 0; i < child_count; i++) {
-                if (shd->pos[i] == AFR_POS_LOCAL) {
-                        ret = i;
-                        break;
-                }
-        }
-        return ret;
+        return;
 }
 
-static int
-_build_index_loc (xlator_t *this, loc_t *loc, char *name, loc_t *parent)
+
+void
+afr_destroy_shd_event_data (void *data)
 {
-        int             ret = 0;
+	shd_event_t *shd_event = data;
 
-        uuid_copy (loc->pargfid, parent->inode->gfid);
-        loc->path = "";
-        loc->name = name;
-        loc->parent = inode_ref (parent->inode);
-        if (!loc->parent) {
-                loc->path = NULL;
-                loc_wipe (loc);
-                ret = -1;
-        }
-        return ret;
+	if (!shd_event)
+		return;
+	GF_FREE (shd_event->path);
+
+        return;
 }
 
-int
-_add_str_to_dict (xlator_t *this, dict_t *output, int child, char *str,
-                  gf_boolean_t dyn)
+
+gf_boolean_t
+afr_shd_is_subvol_local (xlator_t *this, int subvol)
 {
-        //subkey not used for now
-        int             ret = -1;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
+	char *pathinfo = NULL;
+	afr_private_t *priv = NULL;
+	dict_t *xattr = NULL;
+	int ret = 0;
+	gf_boolean_t is_local = _gf_false;
+	loc_t loc = {0, };
 
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
-                goto out;
-        }
+	priv = this->private;
 
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
+	loc.inode = this->itable->root;
+	uuid_copy (loc.gfid, loc.inode->gfid);
 
-        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
-        if (dyn)
-                ret = dict_set_dynstr (output, key, str);
-        else
-                ret = dict_set_str (output, key, str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
-                        str);
-                goto out;
-        }
+	ret = syncop_getxattr (priv->children[subvol], &loc, &xattr,
+			       GF_XATTR_PATHINFO_KEY);
+	if (ret)
+		return _gf_false;
+	if (!xattr)
+		return _gf_false;
 
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_set_uint64 (output, key, count + 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
-                goto out;
-        }
-        ret = 0;
-out:
-        return ret;
-}
+	ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &pathinfo);
+	if (ret)
+		return _gf_false;
 
-int
-_get_path_from_gfid_loc (xlator_t *this, xlator_t *readdir_xl, loc_t *child,
-                         char **fpath)
-{
-        dict_t          *xattr = NULL;
-        char            *path = NULL;
-        int             ret = -1;
+	afr_local_pathinfo (pathinfo, &is_local);
 
-        ret = syncop_getxattr (readdir_xl, child, &xattr,
-                               GFID_TO_PATH_KEY);
-        if (ret)
-                goto out;
-        ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to get path for "
-                        "gfid %s", uuid_utoa (child->gfid));
-                goto out;
-        }
-        path = gf_strdup (path);
-        if (!path) {
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
-out:
-        if (!ret)
-                *fpath = path;
-        if (xattr)
-                dict_unref (xattr);
-        return ret;
+	gf_log (this->name, GF_LOG_DEBUG, "subvol %s is %slocal",
+		priv->children[subvol]->name, is_local? "" : "not ");
+
+	return is_local;
 }
 
+
 int
-_add_event_to_dict (circular_buffer_t *cb, void *data)
+__afr_shd_healer_wait (struct subvol_healer *healer)
 {
-        int               ret = 0;
-        shd_dump_t        *dump_data = NULL;
-        shd_event_t       *shd_event = NULL;
+	afr_private_t *priv = NULL;
+	struct timespec wait_till = {0, };
+	int ret = 0;
 
-        dump_data = data;
-        shd_event = cb->data;
-        if (shd_event->child != dump_data->child)
-                goto out;
-        if (cb->tv.tv_sec >= dump_data->sh_time)
-                ret = _add_str_to_dict (dump_data->this, dump_data->dict,
-                                        dump_data->child, shd_event->path,
-                                        _gf_false);
-out:
-        return ret;
+	priv = healer->this->private;
+
+disabled_loop:
+	wait_till.tv_sec = time (NULL) + 60;
+
+	while (!healer->rerun) {
+		ret = pthread_cond_timedwait (&healer->cond,
+					      &healer->mutex,
+					      &wait_till);
+		if (ret == ETIMEDOUT)
+			break;
+	}
+
+	ret = healer->rerun;
+	healer->rerun = 0;
+
+	if (!priv->shd.enabled)
+		goto disabled_loop;
+
+	return ret;
 }
 
+
 int
-_add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, time_t sh_time,
-                 int child)
+afr_shd_healer_wait (struct subvol_healer *healer)
 {
-        shd_dump_t dump_data = {0};
-
-        dump_data.this = this;
-        dump_data.dict = dict;
-        dump_data.sh_time = sh_time;
-        dump_data.child = child;
-        eh_dump (eh, &dump_data, _add_event_to_dict);
-        return 0;
+	int ret = 0;
+
+	pthread_mutex_lock (&healer->mutex);
+	{
+		ret = __afr_shd_healer_wait (healer);
+	}
+	pthread_mutex_unlock (&healer->mutex);
+
+	return ret;
 }
 
-int
-_add_summary_to_dict (xlator_t *this, afr_crawl_data_t *crawl_data,
-                      gf_dirent_t *entry,
-                      loc_t *childloc, loc_t *parentloc, struct iatt *iattr)
+
+gf_boolean_t
+safe_break (struct subvol_healer *healer)
 {
-        dict_t          *output = NULL;
-        xlator_t        *readdir_xl = NULL;
-        int             ret = -1;
-        char            *path = NULL;
+	gf_boolean_t ret = _gf_false;
 
-        if (uuid_is_null (childloc->gfid))
-                goto out;
+	pthread_mutex_lock (&healer->mutex);
+	{
+		if (healer->rerun)
+			goto unlock;
 
-        output = crawl_data->op_data;
-        readdir_xl = crawl_data->readdir_xl;
+		healer->running = _gf_false;
+		ret = _gf_true;
+	}
+unlock:
+	pthread_mutex_unlock (&healer->mutex);
+
+	return ret;
+}
 
-        ret = _get_path_from_gfid_loc (this, readdir_xl, childloc, &path);
-        if (ret)
-                goto out;
 
-        ret = _add_str_to_dict (this, output, crawl_data->child, path,
-                                _gf_true);
+inode_t *
+afr_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
+{
+	inode_t *inode = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+	struct iatt iatt = {0, };
+
+	inode = inode_find (this->itable, gfid);
+	if (inode)
+		goto out;
+
+	loc.inode = inode_new (this->itable);
+	if (!loc.inode)
+		goto out;
+	uuid_copy (loc.gfid, gfid);
+
+	ret = syncop_lookup (subvol, &loc, NULL, &iatt, NULL, NULL);
+	if (ret < 0)
+		goto out;
+
+	inode = inode_link (loc.inode, NULL, NULL, &iatt);
+	if (inode)
+		inode_lookup (inode);
 out:
-        if (ret && path)
-                GF_FREE (path);
-        return ret;
+	loc_wipe (&loc);
+	return inode;
 }
 
-void
-_remove_stale_index (xlator_t *this, xlator_t *readdir_xl,
-                     loc_t *parent, char *fname)
-{
-        int              ret = 0;
-        loc_t            index_loc = {0};
 
-        ret = _build_index_loc (this, &index_loc, fname, parent);
-        if (ret)
-                goto out;
-        gf_log (this->name, GF_LOG_INFO, "Removing stale index "
-                "for %s on %s", index_loc.name, readdir_xl->name);
-        ret = syncop_unlink (readdir_xl, &index_loc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove"
-                        " index on %s - %s", index_loc.name,
-                        readdir_xl->name, strerror (errno));
-        }
-        index_loc.path = NULL;
-        loc_wipe (&index_loc);
+fd_t *
+afr_shd_index_opendir (xlator_t *this, int child)
+{
+	fd_t *fd = NULL;
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	loc_t rootloc = {0, };
+	inode_t *inode = NULL;
+	int ret = 0;
+	dict_t *xattr = NULL;
+	void *index_gfid = NULL;
+
+	priv = this->private;
+	subvol = priv->children[child];
+
+	rootloc.inode = inode_ref (this->itable->root);
+	uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+
+	ret = syncop_getxattr (subvol, &rootloc, &xattr,
+			       GF_XATTROP_INDEX_GFID);
+	if (ret || !xattr) {
+		errno = -ret;
+		goto out;
+	}
+
+	ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
+	if (ret)
+		goto out;
+
+	gf_log (this->name, GF_LOG_DEBUG, "index-dir gfid for %s: %s",
+		subvol->name, uuid_utoa (index_gfid));
+
+	inode = afr_shd_inode_find (this, subvol, index_gfid);
+	if (!inode)
+		goto out;
+	fd = fd_anonymous (inode);
 out:
-        return;
+	loc_wipe (&rootloc);
+	if (xattr)
+		dict_unref (xattr);
+	return fd;
 }
 
-void
-_crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
-                       int32_t op_ret, int32_t op_errno,
-                       afr_crawl_data_t *crawl_data)
+
+int
+afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
 {
-        int              ret = 0;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        eh_t             *eh = NULL;
-        char             *path = NULL;
-        shd_event_t      *event = NULL;
-
-        priv = this->private;
-        shd  = &priv->shd;
-        if (crawl_data->crawl == INDEX) {
-                if ((op_ret < 0) && (op_errno == ENOENT)) {
-                        _remove_stale_index (this, crawl_data->readdir_xl,
-                                             parent, uuid_utoa (child->gfid));
-                        goto out;
-                }
-                ret = _get_path_from_gfid_loc (this, crawl_data->readdir_xl,
-                                               child, &path);
-                if (ret)
-                        goto out;
-        } else {
-                path = gf_strdup (child->path);
-                if (!path) {
-                        ret = -1;
-                        goto out;
-                }
-        }
+	loc_t loc = {0, };
+	int ret = 0;
 
-        if (op_ret < 0 && op_errno == EIO)
-                eh = shd->split_brain;
-        else if (op_ret < 0)
-                eh = shd->heal_failed;
-        else
-                eh = shd->healed;
-        ret = -1;
-        event = GF_CALLOC (1, sizeof (*event), gf_afr_mt_shd_event_t);
-        if (!event)
-                goto out;
-        event->child = crawl_data->child;
-        event->path = path;
-        ret = eh_save_history (eh, event);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "%s:Failed to save to "
-                        "eh, (%d, %s)", path, op_ret, strerror (op_errno));
-                goto out;
-        }
-        ret = 0;
-out:
-        if (ret && path)
-                GF_FREE (path);
-        return;
+	loc.parent = inode_ref (inode);
+	loc.name = name;
+
+	ret = syncop_unlink (subvol, &loc);
+
+	loc_wipe (&loc);
+	return ret;
 }
 
+
 int
-_self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *entry,
-                  loc_t *child, loc_t *parent, struct iatt *iattr)
+afr_shd_selfheal_name (struct subvol_healer *healer, int child, uuid_t parent,
+		       const char *bname)
 {
-        struct iatt      parentbuf = {0};
-        int              ret = 0;
+	int ret = -1;
 
-        if (uuid_is_null (child->gfid))
-                gf_log (this->name, GF_LOG_DEBUG, "lookup %s", child->path);
-        else
-                gf_log (this->name, GF_LOG_DEBUG, "lookup %s",
-                        uuid_utoa (child->gfid));
+	ret = afr_selfheal_name (THIS, parent, bname);
 
-        ret = syncop_lookup (this, child, NULL,
-                             iattr, NULL, &parentbuf);
-        _crawl_post_sh_action (this, parent, child, ret, errno, crawl_data);
-        return ret;
+	return ret;
 }
 
-static int
-afr_crawl_done  (int ret, call_frame_t *sync_frame, void *data)
+int
+afr_shd_selfheal (struct subvol_healer *healer, int child, uuid_t gfid)
 {
-        GF_FREE (data);
-        STACK_DESTROY (sync_frame->root);
-        return 0;
+	int ret = 0;
+	eh_t *eh = NULL;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	shd_event_t *shd_event = NULL;
+	char *path = NULL;
+	xlator_t *subvol = NULL;
+	xlator_t *this = NULL;
+	crawl_event_t *crawl_event = NULL;
+
+	this = healer->this;
+	priv = this->private;
+	shd = &priv->shd;
+	crawl_event = &healer->crawl_event;
+
+	subvol = priv->children[child];
+
+	ret = afr_selfheal (this, gfid);
+
+	if (ret == -EIO) {
+		eh = shd->split_brain;
+		crawl_event->split_brain_count++;
+	} else if (ret < 0) {
+		eh = shd->heal_failed;
+		crawl_event->heal_failed_count++;
+	} else if (ret == 0) {
+		eh = shd->healed;
+		crawl_event->healed_count++;
+	}
+
+	afr_shd_gfid_to_path (this, subvol, gfid, &path);
+	if (!path)
+		return ret;
+
+	if (eh) {
+		shd_event = GF_CALLOC (1, sizeof(*shd_event),
+				       gf_afr_mt_shd_event_t);
+		if (!shd_event) {
+			GF_FREE (path);
+			return ret;
+		}
+
+		shd_event->child = child;
+		shd_event->path = path;
+
+		if (eh_save_history (eh, shd_event) < 0) {
+			GF_FREE (shd_event);
+			GF_FREE (path);
+		}
+	}
+	return ret;
 }
 
+
 void
-_do_self_heal_on_subvol (xlator_t *this, int child, afr_crawl_type_t crawl)
+afr_shd_sweep_prepare (struct subvol_healer *healer)
 {
-        afr_private_t   *priv = NULL;
-        afr_self_heald_t *shd = NULL;
+	crawl_event_t *event = NULL;
+
+	event = &healer->crawl_event;
 
-        priv = this->private;
-        shd = &priv->shd;
+	event->healed_count = 0;
+	event->split_brain_count = 0;
+	event->heal_failed_count = 0;
 
-        time (&shd->sh_times[child]);
-        afr_start_crawl (this, child, crawl, _self_heal_entry,
-                         NULL, _gf_true, STOP_CRAWL_ON_SINGLE_SUBVOL,
-                         afr_crawl_done);
+	time (&event->start_time);
+	event->end_time = 0;
 }
 
+
 void
-_do_self_heal_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl)
+afr_shd_sweep_done (struct subvol_healer *healer)
 {
-        int             i = 0;
-        afr_private_t   *priv = NULL;
+	crawl_event_t *event = NULL;
+	crawl_event_t *history = NULL;
+	afr_self_heald_t *shd = NULL;
 
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++)
-                _do_self_heal_on_subvol (this, i, INDEX);
-}
+	event = &healer->crawl_event;
+	shd = &(((afr_private_t *)healer->this->private)->shd);
 
-void
-_do_self_heal_on_local_subvol (xlator_t *this, afr_crawl_type_t crawl)
-{
-        int             local_child = -1;
-        afr_private_t   *priv = NULL;
-
-        priv = this->private;
-        local_child = afr_get_local_child (&priv->shd,
-                                           priv->child_count);
-        if (local_child < -1) {
-               gf_log (this->name, GF_LOG_INFO,
-                       "No local bricks found");
-        }
-        _do_self_heal_on_subvol (this, local_child, FULL);
+	time (&event->end_time);
+	history = memdup (event, sizeof (*event));
+	event->start_time = 0;
+
+	if (!history)
+		return;
+
+	if (eh_save_history (shd->statistics[healer->subvol], history) < 0)
+		GF_FREE (history);
 }
 
+
 int
-_get_index_summary_on_local_subvols (xlator_t *this, dict_t *output)
+afr_shd_index_sweep (struct subvol_healer *healer)
 {
-        int             i = 0;
-        afr_private_t   *priv = NULL;
-
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++)
-                afr_start_crawl (this, i, INDEX, _add_summary_to_dict,
-                                 output, _gf_false, 0, NULL);
-        return 0;
+	xlator_t *this = NULL;
+	int child = -1;
+	fd_t *fd = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	uuid_t gfid;
+	int ret = 0;
+	int count = 0;
+
+	this = healer->this;
+	child = healer->subvol;
+	priv = this->private;
+	subvol = priv->children[child];
+
+	fd = afr_shd_index_opendir (this, child);
+	if (!fd) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"unable to opendir index-dir on %s", subvol->name);
+		return -errno;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!priv->shd.enabled) {
+				ret = -EBUSY;
+				break;
+			}
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			gf_log (this->name, GF_LOG_DEBUG, "got entry: %s",
+				entry->d_name);
+
+			ret = uuid_parse (entry->d_name, gfid);
+			if (ret)
+				continue;
+
+			ret = afr_shd_selfheal (healer, child, gfid);
+			if (ret == 0)
+				count++;
+
+			if (ret == -ENOENT || ret == -ESTALE) {
+				afr_shd_index_purge (subvol, fd->inode,
+						     entry->d_name);
+				ret = 0;
+			}
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	if (!ret)
+		ret = count;
+	return ret;
 }
 
+
 int
-_add_all_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
+afr_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
 {
-        afr_private_t           *priv = NULL;
-        afr_self_heald_t        *shd = NULL;
-        int                     i = 0;
+	fd_t *fd = NULL;
+	xlator_t *this = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	int ret = 0;
+
+	this = healer->this;
+	priv = this->private;
+	subvol = priv->children[healer->subvol];
+
+	fd = fd_anonymous (inode);
+	if (!fd)
+		return -errno;
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdirp (subvol, fd, 131072, offset, 0, &entries))) {
+		if (ret < 0)
+			break;
+
+		ret = gf_link_inodes_from_dirent (this, fd->inode, &entries);
+		if (ret)
+			break;
+
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!priv->shd.enabled) {
+				ret = -EBUSY;
+				break;
+			}
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			afr_shd_selfheal_name (healer, healer->subvol,
+					       inode->gfid, entry->d_name);
+
+			afr_shd_selfheal (healer, healer->subvol,
+					  entry->d_stat.ia_gfid);
+
+			if (entry->d_stat.ia_type == IA_IFDIR) {
+				ret = afr_shd_full_sweep (healer, entry->inode);
+				if (ret)
+					break;
+			}
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	return ret;
+}
 
-        priv = this->private;
-        shd = &priv->shd;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (shd->pos[i] != AFR_POS_LOCAL)
-                        continue;
-                _add_eh_to_dict (this, eh, dict, shd->sh_times[i], i);
-        }
-        return 0;
+void *
+afr_shd_index_healer (void *data)
+{
+	struct subvol_healer *healer = NULL;
+	xlator_t *this = NULL;
+	int ret = 0;
+
+	healer = data;
+	THIS = this = healer->this;
+
+	for (;;) {
+		afr_shd_healer_wait (healer);
+
+		ASSERT_LOCAL(this, healer);
+
+		do {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"starting index sweep on subvol %s",
+				afr_subvol_name (this, healer->subvol));
+
+			afr_shd_sweep_prepare (healer);
+
+			ret = afr_shd_index_sweep (healer);
+
+			afr_shd_sweep_done (healer);
+			/*
+			  As long as at least one gfid was
+			  healed, keep retrying. We may have
+			  just healed a directory and thereby
+			  created entries for other gfids which
+			  could not be healed thus far.
+			*/
+
+			gf_log (this->name, GF_LOG_DEBUG,
+				"finished index sweep on subvol %s",
+				afr_subvol_name (this, healer->subvol));
+			/*
+			  Give a pause before retrying to avoid a busy loop
+			  in case the only entry in index is because of
+			  an ongoing I/O.
+			*/
+			sleep (1);
+		} while (ret > 0);
+	}
+
+	return NULL;
 }
 
-int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+
+void *
+afr_shd_full_healer (void *data)
 {
-        gf_xl_afr_op_t   op = GF_AFR_OP_INVALID;
-        int              ret = 0;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        int              xl_id = 0;
+	struct subvol_healer *healer = NULL;
+	xlator_t *this = NULL;
+	int run = 0;
 
-        priv = this->private;
-        shd = &priv->shd;
+	healer = data;
+	THIS = this = healer->this;
 
-        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
-        switch (op) {
-        case GF_AFR_OP_HEAL_INDEX:
-                _do_self_heal_on_local_subvols (this, INDEX);
-                ret = 0;
-                break;
-        case GF_AFR_OP_HEAL_FULL:
-                _do_self_heal_on_local_subvol (this, FULL);
-                ret = 0;
-                break;
-        case GF_AFR_OP_INDEX_SUMMARY:
-                ret = _get_index_summary_on_local_subvols (this, output);
-                if (ret)
-                        goto out;
-                break;
-        case GF_AFR_OP_HEALED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->healed, output);
-                break;
-        case GF_AFR_OP_HEAL_FAILED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->heal_failed,
-                                                   output);
-                break;
-        case GF_AFR_OP_SPLIT_BRAIN_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->split_brain,
-                                                   output);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
-                break;
-        }
-out:
-        dict_del (output, this->name);
-        return ret;
-}
+	for (;;) {
+		pthread_mutex_lock (&healer->mutex);
+		{
+			run = __afr_shd_healer_wait (healer);
+			if (!run)
+				healer->running = _gf_false;
+		}
+		pthread_mutex_unlock (&healer->mutex);
 
-void
-afr_do_poll_self_heal (void *data)
-{
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        struct timeval   timeout = {0};
-        xlator_t         *this = NULL;
-        long             child = (long)data;
-
-        this = THIS;
-        priv = this->private;
-        shd = &priv->shd;
-
-        if (shd->enabled)
-                _do_self_heal_on_subvol (this, child, INDEX);
-        if (shd->pos[child] == AFR_POS_REMOTE)
-                goto out;
-        timeout.tv_sec = AFR_POLL_TIMEOUT;
-        timeout.tv_usec = 0;
-        if (shd->timer[child])
-                gf_timer_call_cancel (this->ctx, shd->timer[child]);
-        shd->timer[child] = gf_timer_call_after (this->ctx, timeout,
-                                                 afr_do_poll_self_heal, data);
-
-        if (shd->timer[child] == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Cannot create pending self-heal polling timer for %s",
-                        priv->children[child]->name);
-        }
-out:
-        return;
-}
+		if (!run)
+			break;
 
-static int
-get_pathinfo_host (char *pathinfo, char *hostname, size_t size)
-{
-        char    *start = NULL;
-        char    *end = NULL;
-        int     ret  = -1;
-        int     i    = 0;
+		ASSERT_LOCAL(this, healer);
 
-        if (!pathinfo)
-                goto out;
+		gf_log (this->name, GF_LOG_INFO,
+			"starting full sweep on subvol %s",
+			afr_subvol_name (this, healer->subvol));
 
-        start = strchr (pathinfo, ':');
-        if (!start)
-                goto out;
-        end = strrchr (pathinfo, ':');
-        if (start == end)
-                goto out;
+		afr_shd_sweep_prepare (healer);
 
-        memset (hostname, 0, size);
-        i = 0;
-        while (++start != end)
-                hostname[i++] = *start;
-        ret = 0;
-out:
-        return ret;
+		afr_shd_full_sweep (healer, this->itable->root);
+
+		afr_shd_sweep_done (healer);
+
+		gf_log (this->name, GF_LOG_INFO,
+			"finished full sweep on subvol %s",
+			afr_subvol_name (this, healer->subvol));
+	}
+
+	return NULL;
 }
 
+
 int
-afr_local_pathinfo (char *pathinfo, gf_boolean_t *local)
+afr_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
 {
-        int             ret   = 0;
-        char            pathinfohost[1024] = {0};
-        char            localhost[1024] = {0};
-        xlator_t        *this = THIS;
+	int ret = 0;
 
-        *local = _gf_false;
-        ret = get_pathinfo_host (pathinfo, pathinfohost, sizeof (pathinfohost));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid pathinfo: %s",
-                        pathinfo);
-                goto out;
-        }
+	ret = pthread_mutex_init (&healer->mutex, NULL);
+	if (ret)
+		goto out;
 
-        ret = gethostname (localhost, sizeof (localhost));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "gethostname() failed, "
-                        "reason: %s", strerror (errno));
-                goto out;
-        }
+	ret = pthread_cond_init (&healer->cond, NULL);
+	if (ret)
+		goto out;
 
-        if (!strcmp (localhost, pathinfohost))
-                *local = _gf_true;
+	healer->this = this;
+	healer->running = _gf_false;
+	healer->rerun = _gf_false;
+	healer->local = _gf_false;
 out:
-        return ret;
+	return ret;
 }
 
 
 int
-afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
-                           loc_t *dirloc)
+afr_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
+		      void *(threadfn)(void *))
 {
-        afr_private_t *priv = NULL;
-        dict_t        *xattr = NULL;
-        void          *index_gfid = NULL;
-        loc_t         rootloc = {0};
-        struct iatt   iattr = {0};
-        struct iatt   parent = {0};
-        int           ret = 0;
-        xlator_t      *readdir_xl = crawl_data->readdir_xl;
-
-        priv = this->private;
-        if (crawl_data->crawl == FULL) {
-                afr_build_root_loc (this, dirloc);
-        } else {
-                afr_build_root_loc (this, &rootloc);
-                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
-                                       GF_XATTROP_INDEX_GFID);
-                if (ret < 0)
-                        goto out;
-                ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get index "
-                                "dir gfid on %s", readdir_xl->name);
-                        goto out;
-                }
-                if (!index_gfid) {
-                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
-                                "on %s", readdir_xl->name);
-                        ret = -1;
-                        goto out;
-                }
-                uuid_copy (dirloc->gfid, index_gfid);
-                dirloc->path = "";
-                dirloc->inode = inode_new (priv->root_inode->table);
-                ret = syncop_lookup (readdir_xl, dirloc, NULL,
-                                     &iattr, NULL, &parent);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "lookup failed on "
-                                "index dir on %s", readdir_xl->name);
-                        goto out;
-                }
-                inode_link (dirloc->inode, NULL, NULL, &iattr);
-        }
-        ret = 0;
-out:
-        if (xattr)
-                dict_unref (xattr);
-        loc_wipe (&rootloc);
-        return ret;
+	int ret = 0;
+
+	pthread_mutex_lock (&healer->mutex);
+	{
+		if (healer->running) {
+			pthread_cond_signal (&healer->cond);
+		} else {
+			ret = gf_thread_create (&healer->thread, NULL,
+						threadfn, healer);
+			if (ret)
+				goto unlock;
+			healer->running = 1;
+		}
+
+		healer->rerun = 1;
+	}
+unlock:
+	pthread_mutex_unlock (&healer->mutex);
+
+	return ret;
 }
 
+
 int
-afr_crawl_opendir (xlator_t *this, afr_crawl_data_t *crawl_data, fd_t **dirfd,
-                   loc_t *dirloc)
+afr_shd_full_healer_spawn (xlator_t *this, int subvol)
 {
-        fd_t          *fd   = NULL;
-        int           ret = 0;
-
-        if (crawl_data->crawl == FULL) {
-                fd = fd_create (dirloc->inode, crawl_data->pid);
-                if (!fd) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to create fd for %s", dirloc->path);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_opendir (crawl_data->readdir_xl, dirloc, fd);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "opendir failed on %s", dirloc->path);
-                        goto out;
-                }
-        } else {
-                fd = fd_anonymous (dirloc->inode);
-        }
-        ret = 0;
-out:
-        if (!ret)
-                *dirfd = fd;
-        return ret;
+	return afr_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
+				     afr_shd_full_healer);
 }
 
-xlator_t*
-afr_crawl_readdir_xl_get (xlator_t *this, afr_crawl_data_t *crawl_data)
-{
-        afr_private_t *priv = this->private;
-
-        if (crawl_data->crawl == FULL) {
-                return this;
-        } else {
-                return priv->children[crawl_data->child];
-        }
-        return NULL;
-}
 
 int
-afr_crawl_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent,
-                           gf_dirent_t *entry, afr_crawl_data_t *crawl_data)
+afr_shd_index_healer_spawn (xlator_t *this, int subvol)
 {
-        int           ret = 0;
-        afr_private_t *priv = NULL;
-
-        priv = this->private;
-        if (crawl_data->crawl == FULL) {
-                ret = afr_build_child_loc (this, child, parent, entry->d_name);
-        } else {
-                child->path = "";
-                child->inode = inode_new (priv->root_inode->table);
-                uuid_parse (entry->d_name, child->gfid);
-        }
-        return ret;
+	return afr_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
+				     afr_shd_index_healer);
 }
 
-gf_boolean_t
-_crawl_proceed (xlator_t *this, int child, int crawl_flags)
+
+int
+afr_shd_dict_add_crawl_event (xlator_t *this, dict_t *output,
+			      crawl_event_t *crawl_event)
 {
-        afr_private_t *priv = this->private;
-        gf_boolean_t proceed = _gf_false;
+        int             ret = 0;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = 0;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+	int             child = -1;
 
-        if (!priv->child_up[child]) {
-                gf_log (this->name, GF_LOG_ERROR, "Stopping crawl for %s "
-                        ", subvol went down", priv->children[child]->name);
-                goto out;
-        }
+	child = crawl_event->child;
+        healed_count = crawl_event->healed_count;
+        split_brain_count = crawl_event->split_brain_count;
+        heal_failed_count = crawl_event->heal_failed_count;
+        crawl_type = crawl_event->crawl_type;
 
-        if (crawl_flags & STOP_CRAWL_ON_SINGLE_SUBVOL) {
-                if (afr_up_children_count (priv->child_up,
-                                           priv->child_count) < 2) {
-                        gf_log (this->name, GF_LOG_ERROR, "Stopping crawl as "
-                                "< 2 children are up");
-                        goto out;
-                }
-        }
-        proceed = _gf_true;
-out:
-        return proceed;
-}
+	if (!crawl_event->start_time)
+		goto out;
 
-static int
-_process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
-                  off_t *offset, afr_crawl_data_t *crawl_data)
-{
-        gf_dirent_t      *entry = NULL;
-        gf_dirent_t      *tmp = NULL;
-        int              ret = 0;
-        loc_t            entry_loc = {0};
-        fd_t             *fd = NULL;
-        struct iatt      iattr = {0};
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if (!_crawl_proceed (this, crawl_data->child,
-                                     crawl_data->crawl_flags)) {
-                        ret = -1;
-                        goto out;
-                }
-                *offset = entry->d_off;
-                if (IS_ENTRY_CWD (entry->d_name) ||
-                    IS_ENTRY_PARENT (entry->d_name))
-                        continue;
-                if ((crawl_data->crawl == FULL) &&
-                     uuid_is_null (entry->d_stat.ia_gfid)) {
-                        gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
-                                "gfid present skipping",
-                                parentloc->path, entry->d_name);
-                        continue;
-                }
-
-                if (crawl_data->crawl == INDEX)
-                        entry_loc.path = NULL;//HACK
-                loc_wipe (&entry_loc);
-                ret = afr_crawl_build_child_loc (this, &entry_loc, parentloc,
-                                                 entry, crawl_data);
-                if (ret)
-                        goto out;
+        start_time_str = gf_strdup (ctime (&crawl_event->start_time));
 
-                ret = crawl_data->process_entry (this, crawl_data, entry,
-                                                 &entry_loc, parentloc, &iattr);
+	if (crawl_event->end_time)
+		end_time_str = gf_strdup (ctime (&crawl_event->end_time));
 
-                if (crawl_data->crawl == INDEX)
-                        continue;
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
 
-                if (ret || !IA_ISDIR (iattr.ia_type))
-                        continue;
+        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
 
-                inode_link (entry_loc.inode, parentloc->inode, NULL, &iattr);
 
-                fd = NULL;
-                ret = afr_crawl_opendir (this, crawl_data, &fd, &entry_loc);
-                if (ret)
-                        continue;
-                ret = _crawl_directory (fd, &entry_loc, crawl_data);
-                if (fd)
-                        fd_unref (fd);
+        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64(output, key, healed_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_healed_count to outout");
+                goto out;
+	}
+
+        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, split_brain_count);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_split_brain_count to outout");
+                goto out;
         }
-        ret = 0;
-out:
-        if (crawl_data->crawl == INDEX)
-                entry_loc.path = NULL;
-        if (entry_loc.path)
-                loc_wipe (&entry_loc);
-        return ret;
-}
 
-static int
-_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data)
-{
-        xlator_t        *this = NULL;
-        off_t           offset   = 0;
-        gf_dirent_t     entries;
-        int             ret = 0;
-        gf_boolean_t    free_entries = _gf_false;
-        xlator_t        *readdir_xl = crawl_data->readdir_xl;
+        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_str (output, key, crawl_type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_type to output");
+                goto out;
+        }
 
-        INIT_LIST_HEAD (&entries.list);
-        this = THIS;
+        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, heal_failed_count);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_healed_failed_count to outout");
+                goto out;
+        }
 
-        GF_ASSERT (loc->inode);
+        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, start_time_str);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_start_time to outout");
+                goto out;
+        } else {
+		start_time_str = NULL;
+	}
 
-        if (loc->path)
-                gf_log (this->name, GF_LOG_DEBUG, "crawling %s", loc->path);
+	if (!end_time_str)
+                progress = 1;
         else
-                gf_log (this->name, GF_LOG_DEBUG, "crawling %s",
-                        uuid_utoa (loc->gfid));
-
-        while (1) {
-                if (crawl_data->crawl == FULL)
-                        ret = syncop_readdirp (readdir_xl, fd, 131072, offset,
-                                               NULL, &entries);
-                else
-                        ret = syncop_readdir (readdir_xl, fd, 131072, offset,
-                                              &entries);
-                if (ret <= 0)
-                        break;
-                ret = 0;
-                free_entries = _gf_true;
-
-                if (!_crawl_proceed (this, crawl_data->child,
-                                     crawl_data->crawl_flags)) {
-                        ret = -1;
-                        goto out;
-                }
-                if (list_empty (&entries.list))
-                        goto out;
+                progress = 0;
+
+        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        if (!end_time_str)
+                end_time_str = gf_strdup ("Could not determine the end time");
+        ret = dict_set_dynstr (output, key, end_time_str);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_end_time to outout");
+                goto out;
+        } else {
+		end_time_str = NULL;
+	}
+
+        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
+                  xl_id, child, count);
 
-                ret = _process_entries (this, loc, &entries, &offset,
-                                        crawl_data);
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
+        ret = dict_set_int32 (output, key, progress);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_inprogress to outout");
+                goto out;
         }
-        ret = 0;
+
+	snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+	ret = dict_set_uint64 (output, key, count + 1);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not increment the counter.");
+                goto out;
+	}
 out:
-        if (free_entries)
-                gf_dirent_free (&entries);
+	GF_FREE (start_time_str);
+	GF_FREE (end_time_str);
         return ret;
 }
 
-static char*
-position_str_get (afr_child_pos_t pos)
-{
-        switch (pos) {
-        case AFR_POS_UNKNOWN:
-                return "unknown";
-        case AFR_POS_LOCAL:
-                return "local";
-        case AFR_POS_REMOTE:
-                return "remote";
-        }
-        return NULL;
-}
 
 int
-afr_find_child_position (xlator_t *this, int child)
+afr_shd_dict_add_path (xlator_t *this, dict_t *output, int child, char *path,
+		       struct timeval *tv)
 {
-        afr_private_t    *priv = NULL;
-        dict_t           *xattr_rsp = NULL;
-        loc_t            loc = {0};
-        int              ret = 0;
-        gf_boolean_t     local = _gf_false;
-        char             *pathinfo = NULL;
-        afr_child_pos_t  *pos = NULL;
-
-        priv = this->private;
-        pos = &priv->shd.pos[child];
-
-        if (!priv->root_inode) {
-                LOCK (&priv->lock);
-                {
-                        if (!priv->root_inode)
-                                priv->root_inode = inode_ref
-                                                       (this->itable->root);
-                }
-                UNLOCK (&priv->lock);
-        }
-
-        afr_build_root_loc (this, &loc);
+        int             ret = -1;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
 
-        ret = syncop_getxattr (priv->children[child], &loc, &xattr_rsp,
-                               GF_XATTR_PATHINFO_KEY);
+        ret = dict_get_int32 (output, this->name, &xl_id);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getxattr failed on child "
-                        "%d", child);
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
                 goto out;
         }
 
-        ret = dict_get_str (xattr_rsp, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
+	ret = dict_set_dynstr (output, key, path);
+
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Pathinfo key not found on "
-                        "child %d", child);
+                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
+                        path);
                 goto out;
         }
 
-        ret = afr_local_pathinfo (pathinfo, &local);
-        if (ret)
+	if (tv) {
+		snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
+			  child, count);
+		ret = dict_set_uint32 (output, key, tv->tv_sec);
+		if (ret) {
+			gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
+				path);
+			goto out;
+		}
+	}
+
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+
+        ret = dict_set_uint64 (output, key, count + 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
                 goto out;
-        if (local)
-                *pos = AFR_POS_LOCAL;
-        else
-                *pos = AFR_POS_REMOTE;
+        }
 
-        gf_log (this->name, GF_LOG_INFO, "child %s is %s",
-                priv->children[child]->name, position_str_get (*pos));
+        ret = 0;
 out:
-        if (ret)
-                *pos = AFR_POS_UNKNOWN;
-        loc_wipe (&loc);
         return ret;
 }
 
-static inline int
-afr_is_local_child (afr_self_heald_t *shd, int child, unsigned int child_count)
+
+int
+afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid, char **path_p)
 {
-        return (shd->pos[child] == AFR_POS_LOCAL);
+	loc_t loc = {0,};
+	char *path = NULL;
+	dict_t *xattr = NULL;
+	int ret = 0;
+
+	uuid_copy (loc.gfid, gfid);
+	loc.inode = inode_new (this->itable);
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, GFID_TO_PATH_KEY);
+	loc_wipe (&loc);
+	if (ret)
+		return ret;
+
+	ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
+	if (ret || !path)
+		return -EINVAL;
+
+	*path_p = gf_strdup (path);
+	if (!*path_p)
+		return -ENOMEM;
+	return 0;
 }
 
-static int
-afr_dir_crawl (void *data)
+
+int
+afr_shd_gather_index_entries (xlator_t *this, int child, dict_t *output)
 {
-        xlator_t            *this = NULL;
-        afr_private_t       *priv = NULL;
-        afr_self_heald_t    *shd = NULL;
-        int                 ret = -1;
-        xlator_t            *readdir_xl = NULL;
-        fd_t                *fd = NULL;
-        loc_t               dirloc = {0};
-        afr_crawl_data_t    *crawl_data = data;
-
-        this = THIS;
-        priv = this->private;
-        shd = &priv->shd;
-
-        if (!_crawl_proceed (this, crawl_data->child, crawl_data->crawl_flags))
-                goto out;
+	fd_t *fd = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	uuid_t gfid;
+	int ret = 0;
+	int count = 0;
+	char *path = NULL;
+
+	priv = this->private;
+	subvol = priv->children[child];
+
+	fd = afr_shd_index_opendir (this, child);
+	if (!fd) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"unable to opendir index-dir on %s", subvol->name);
+		return -errno;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			gf_log (this->name, GF_LOG_DEBUG, "got entry: %s",
+				entry->d_name);
+
+			ret = uuid_parse (entry->d_name, gfid);
+			if (ret)
+				continue;
+
+			path = NULL;
+			ret = afr_shd_gfid_to_path (this, subvol, gfid, &path);
+
+			if (ret == -ENOENT || ret == -ESTALE) {
+				afr_shd_index_purge (subvol, fd->inode,
+						     entry->d_name);
+				ret = 0;
+				continue;
+			}
+
+			ret = afr_shd_dict_add_path (this, output, child, path,
+						     NULL);
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	if (!ret)
+		ret = count;
+	return ret;
+}
 
-        ret = afr_find_child_position (this, crawl_data->child);
-        if (ret)
-                goto out;
 
-        if (!afr_is_local_child (shd, crawl_data->child, priv->child_count))
-                goto out;
+int
+afr_add_shd_event (circular_buffer_t *cb, void *data)
+{
+	dict_t *output = NULL;
+	xlator_t *this = THIS;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	shd_event_t *shd_event = NULL;
+	char *path = NULL;
+
+	output = data;
+	priv = this->private;
+	shd = &priv->shd;
+	shd_event = cb->data;
+
+	if (!shd->index_healers[shd_event->child].local)
+		return 0;
+
+	path = gf_strdup (shd_event->path);
+	if (!path)
+		return -ENOMEM;
+
+	afr_shd_dict_add_path (this, output, shd_event->child, path,
+			       &cb->tv);
+	return 0;
+}
 
-        readdir_xl = afr_crawl_readdir_xl_get (this, crawl_data);
-        if (!readdir_xl)
-                goto out;
-        crawl_data->readdir_xl = readdir_xl;
+int
+afr_add_crawl_event (circular_buffer_t *cb, void *data)
+{
+	dict_t *output = NULL;
+	xlator_t *this = THIS;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	crawl_event_t *crawl_event = NULL;
 
-        ret = afr_crawl_build_start_loc (this, crawl_data, &dirloc);
-        if (ret)
-                goto out;
+	output = data;
+	priv = this->private;
+	shd = &priv->shd;
+	crawl_event = cb->data;
 
-        ret = afr_crawl_opendir (this, crawl_data, &fd, &dirloc);
-        if (ret)
-                goto out;
+	if (!shd->index_healers[crawl_event->child].local)
+		return 0;
 
-        ret = _crawl_directory (fd, &dirloc, crawl_data);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Crawl failed on %s",
-                        readdir_xl->name);
-        else
-                gf_log (this->name, GF_LOG_INFO, "Crawl completed "
-                        "on %s", readdir_xl->name);
-        if (crawl_data->crawl == INDEX)
-                dirloc.path = NULL;
-out:
-        if (fd)
-                fd_unref (fd);
-        if (crawl_data->crawl == INDEX)
-                dirloc.path = NULL;
-        loc_wipe (&dirloc);
-        return ret;
+	afr_shd_dict_add_crawl_event (this, output, crawl_event);
+
+	return 0;
 }
 
-static int
-afr_dir_exclusive_crawl (void *data)
-{
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        gf_boolean_t     crawl = _gf_false;
-        int              ret = 0;
-        int              child = -1;
-        xlator_t         *this = NULL;
-        afr_crawl_data_t *crawl_data = data;
-
-        this = THIS;
-        priv = this->private;
-        shd = &priv->shd;
-        child = crawl_data->child;
-
-        LOCK (&priv->lock);
-        {
-                if (shd->inprogress[child]) {
-                        if (shd->pending[child] != FULL)
-                                shd->pending[child] = crawl_data->crawl;
-                } else {
-                        shd->inprogress[child] = _gf_true;
-                        crawl = _gf_true;
-                }
-        }
-        UNLOCK (&priv->lock);
 
-        if (!crawl) {
-                gf_log (this->name, GF_LOG_INFO, "Another crawl is in progress "
-                        "for %s", priv->children[child]->name);
+int
+afr_selfheal_daemon_init (xlator_t *this)
+{
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	int ret = -1;
+	int i = 0;
+
+	priv = this->private;
+	shd = &priv->shd;
+
+	this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
+	if (!this->itable)
+		goto out;
+
+	shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
+					priv->child_count,
+					gf_afr_mt_subvol_healer_t);
+	if (!shd->index_healers)
+		goto out;
+
+	for (i = 0; i < priv->child_count; i++) {
+		shd->index_healers[i].subvol = i;
+		ret = afr_shd_healer_init (this, &shd->index_healers[i]);
+		if (ret)
+			goto out;
+	}
+
+	shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
+				       priv->child_count,
+				       gf_afr_mt_subvol_healer_t);
+	if (!shd->full_healers)
+		goto out;
+	for (i = 0; i < priv->child_count; i++) {
+		shd->full_healers[i].subvol = i;
+		ret = afr_shd_healer_init (this, &shd->full_healers[i]);
+		if (ret)
+			goto out;
+	}
+
+	shd->healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false,
+			      afr_destroy_shd_event_data);
+        if (!shd->healed)
+		goto out;
+
+	shd->heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false,
+				   afr_destroy_shd_event_data);
+	if (!shd->heal_failed)
+		goto out;
+
+	shd->split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+				   afr_destroy_shd_event_data);
+	if (!shd->split_brain)
+		goto out;
+
+        shd->statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
+				     gf_common_mt_eh_t);
+        if (!shd->statistics)
                 goto out;
+
+        for (i = 0; i < priv->child_count ; i++) {
+                shd->statistics[i] = eh_new (AFR_STATISTICS_HISTORY_SIZE,
+					     _gf_false,
+					     afr_destroy_crawl_event_data);
+                if (!shd->statistics[i])
+                        goto out;
+		shd->full_healers[i].crawl_event.child = i;
+		shd->full_healers[i].crawl_event.crawl_type = "FULL";
+		shd->index_healers[i].crawl_event.child = i;
+		shd->index_healers[i].crawl_event.crawl_type = "INDEX";
         }
 
-        do {
-                afr_dir_crawl (data);
-                LOCK (&priv->lock);
-                {
-                        if (shd->pending[child] != NONE) {
-                                crawl_data->crawl = shd->pending[child];
-                                shd->pending[child] = NONE;
-                        } else {
-                                shd->inprogress[child] = _gf_false;
-                                crawl = _gf_false;
-                        }
-                }
-                UNLOCK (&priv->lock);
-        } while (crawl);
+	ret = 0;
 out:
-        return ret;
+	return ret;
 }
 
-void
-afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
-                 process_entry_cbk_t process_entry, void *op_data,
-                 gf_boolean_t exclusive, int crawl_flags,
-                 afr_crawl_done_cbk_t crawl_done)
-{
-        afr_private_t              *priv = NULL;
-        afr_self_heald_t           *shd = NULL;
-        call_frame_t               *frame = NULL;
-        afr_crawl_data_t           *crawl_data = NULL;
-        int                        ret = 0;
-        int (*crawler) (void*) = NULL;
-
-        priv = this->private;
-        shd = &priv->shd;
-        if (!shd->enabled)
-                goto out;
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+int
+afr_selfheal_childup (xlator_t *this, int subvol)
+{
+	afr_shd_index_healer_spawn (this, subvol);
 
-        afr_set_lk_owner (frame, this);
-        afr_set_low_priority (frame);
-        crawl_data = GF_CALLOC (1, sizeof (*crawl_data),
-                                gf_afr_mt_crawl_data_t);
-        if (!crawl_data)
-                goto out;
-        crawl_data->process_entry = process_entry;
-        crawl_data->child = idx;
-        crawl_data->pid = frame->root->pid;
-        crawl_data->crawl = crawl;
-        crawl_data->op_data = op_data;
-        crawl_data->crawl_flags = crawl_flags;
-        gf_log (this->name, GF_LOG_INFO, "starting crawl %d for %s",
-                crawl_data->crawl, priv->children[idx]->name);
-
-        if (exclusive)
-                crawler = afr_dir_exclusive_crawl;
-        else
-                crawler = afr_dir_crawl;
-        ret = synctask_new (this->ctx->env, crawler,
-                            crawl_done, frame, crawl_data);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Could not create the "
-                        "task for %d ret %d", idx, ret);
-out:
-        return;
+	return 0;
 }
 
-void
-afr_build_root_loc (xlator_t *this, loc_t *loc)
-{
-        afr_private_t   *priv = NULL;
 
-        priv = this->private;
-        loc->path = gf_strdup ("/");
-        loc->name = "";
-        loc->inode = inode_ref (priv->root_inode);
-        uuid_copy (loc->gfid, loc->inode->gfid);
+int64_t
+afr_shd_get_index_count (xlator_t *this, int i)
+{
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	uint64_t count = 0;
+	loc_t rootloc = {0, };
+	dict_t *xattr = NULL;
+	int ret = -1;
+
+	priv = this->private;
+	subvol = priv->children[i];
+
+	rootloc.inode = inode_ref (this->itable->root);
+	uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+
+	ret = syncop_getxattr (subvol, &rootloc, &xattr,
+			       GF_XATTROP_INDEX_COUNT);
+	loc_wipe (&rootloc);
+
+	if (ret < 0)
+		return -1;
+
+	ret = dict_get_uint64 (xattr, GF_XATTROP_INDEX_COUNT, &count);
+	if (ret)
+		return -1;
+	return count;
 }
 
+
 int
-afr_set_root_gfid (dict_t *dict)
+afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
 {
-        uuid_t gfid;
-        int ret = 0;
+        gf_xl_afr_op_t   op = GF_AFR_OP_INVALID;
+        int              ret = 0;
+        int              xl_id = 0;
+	afr_private_t   *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	struct subvol_healer *healer = NULL;
+	int i = 0;
+	char key[64];
+	int op_ret = 0;
+	int64_t cnt = 0;
 
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
+	priv = this->private;
+	shd = &priv->shd;
 
-        ret = afr_set_dict_gfid (dict, gfid);
+	for (i = 0; i < priv->child_count; i++)
+		if (priv->child_up[i] == -1)
+			goto out;
 
-        return ret;
-}
+        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (input, this->name, &xl_id);
+        if (ret)
+                goto out;
+        ret = dict_set_int32 (output, this->name, xl_id);
+        if (ret)
+                goto out;
+        switch (op) {
+        case GF_AFR_OP_HEAL_INDEX:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			healer = &shd->index_healers[i];
+			snprintf (key, 64, "%d-%d-status", xl_id, i);
+
+			if (!priv->child_up[i]) {
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else if (AFR_COUNT (priv->child_up,
+					      priv->child_count) < 2) {
+				ret = dict_set_str (output, key,
+						    "< 2 bricks in replica are up");
+			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
+				ret = dict_set_str (output, key,
+						    "Brick is remote");
+			} else {
+				ret = dict_set_str (output, key,
+						    "Started self-heal");
+				afr_shd_index_healer_spawn (this, i);
+				op_ret = 0;
+			}
+		}
+                break;
+        case GF_AFR_OP_HEAL_FULL:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			healer = &shd->full_healers[i];
+			snprintf (key, 64, "%d-%d-status", xl_id, i);
+
+			if (!priv->child_up[i]) {
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else if (AFR_COUNT (priv->child_up,
+					      priv->child_count) < 2) {
+				ret = dict_set_str (output, key,
+						    "< 2 bricks in replica are up");
+			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
+				ret = dict_set_str (output, key,
+						    "Brick is remote");
+			} else {
+				ret = dict_set_str (output, key,
+						    "Started self-heal");
+				afr_shd_full_healer_spawn (this, i);
+				op_ret = 0;
+			}
+		}
+                break;
+        case GF_AFR_OP_INDEX_SUMMARY:
+		for (i = 0; i < priv->child_count; i++)
+			if (shd->index_healers[i].local)
+				afr_shd_gather_index_entries (this, i, output);
+                break;
+        case GF_AFR_OP_HEALED_FILES:
+		eh_dump (shd->healed, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_HEAL_FAILED_FILES:
+		eh_dump (shd->heal_failed, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_SPLIT_BRAIN_FILES:
+		eh_dump (shd->split_brain, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_STATISTICS:
+		for (i = 0; i < priv->child_count; i++) {
+			eh_dump (shd->statistics[i], output,
+				 afr_add_crawl_event);
+			afr_shd_dict_add_crawl_event (this, output,
+						      &shd->index_healers[i].crawl_event);
+			afr_shd_dict_add_crawl_event (this, output,
+						      &shd->full_healers[i].crawl_event);
+		}
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			if (!priv->child_up[i]) {
+				snprintf (key, 64, "%d-%d-status", xl_id, i);
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else {
+				snprintf (key, 64, "%d-%d-hardlinks", xl_id, i);
+				cnt = afr_shd_get_index_count (this, i);
+				if (cnt >= 0) {
+					ret = dict_set_uint64 (output, key, cnt);
+				}
+				op_ret = 0;
+			}
+		}
+
+//                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
+//                                                     STATISTICS_TO_BE_HEALED,
+//                                                     output);
+                break;
 
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
+                break;
+        }
+out:
+        dict_del (output, this->name);
+        return op_ret;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index 57b29ce2d..10e229ee7 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -1,56 +1,72 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef __AFR_SELF_HEALD_H__
-#define __AFR_SELF_HEALD_H__
-#include "xlator.h"
 
-#define IS_ROOT_PATH(path) (!strcmp (path, "/"))
-#define IS_ENTRY_CWD(entry) (!strcmp (entry, "."))
-#define IS_ENTRY_PARENT(entry) (!strcmp (entry, ".."))
-#define AFR_ALL_CHILDREN -1
+#ifndef _AFR_SELF_HEALD_H
+#define _AFR_SELF_HEALD_H
+
+#include <pthread.h>
+
 
-typedef struct afr_crawl_data_ {
-        int                 child;
-        pid_t               pid;
-        afr_crawl_type_t    crawl;
-        xlator_t            *readdir_xl;
-        void                *op_data;
-        int                 crawl_flags;
-        int (*process_entry) (xlator_t *this, struct afr_crawl_data_ *crawl_data,
-                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
-                              struct iatt *iattr);
-} afr_crawl_data_t;
+typedef struct {
+	int child;
+	char *path;
+} shd_event_t;
 
-typedef int (*process_entry_cbk_t) (xlator_t *this, afr_crawl_data_t *crawl_data,
-                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
-                              struct iatt *iattr);
+typedef struct {
+	int      child;
+	uint64_t healed_count;
+        uint64_t split_brain_count;
+        uint64_t heal_failed_count;
 
-void afr_proactive_self_heal (xlator_t *this, int idx);
+	/* If start_time is 0, it means crawler is not in progress
+	   and stats are not valid */
+	time_t   start_time;
+	/* If start_time is NOT 0 and end_time is 0, it means
+	   cralwer is in progress */
+        time_t   end_time;
+        char     *crawl_type;
+} crawl_event_t;
 
-void afr_build_root_loc (xlator_t *this, loc_t *loc);
+struct subvol_healer {
+	xlator_t        *this;
+	int              subvol;
+	gf_boolean_t     local;
+	gf_boolean_t     running;
+	gf_boolean_t     rerun;
+	crawl_event_t    crawl_event;
+	pthread_mutex_t  mutex;
+	pthread_cond_t   cond;
+	pthread_t        thread;
+};
 
-int afr_set_root_gfid (dict_t *dict);
+typedef struct {
+	gf_boolean_t            iamshd;
+	gf_boolean_t            enabled;
+	struct subvol_healer   *index_healers;
+	struct subvol_healer   *full_healers;
+
+	eh_t                    *healed;
+        eh_t                    *heal_failed;
+        eh_t                    *split_brain;
+        eh_t                    **statistics;
+} afr_self_heald_t;
 
-void
-afr_do_poll_self_heal (void *data);
+
+int
+afr_selfheal_childup (xlator_t *this, int subvol);
+
+int
+afr_selfheal_daemon_init (xlator_t *this);
 
 int
 afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
-#endif /* __AFR_SELF_HEALD_H__ */
+
+#endif /* !_AFR_SELF_HEALD_H */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 36e2812f9..205ff759e 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -1,215 +1,173 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
 #include "byte-order.h"
 #include "common-utils.h"
+#include "timer.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
 
 #include <signal.h>
 
+gf_boolean_t
+afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this);
 
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path
-                                      of RENAME */
-#define LOCKED_LOWER    0x2        /* for lower_path of RENAME */
+gf_boolean_t
+afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this);
 
+int
+afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
+		  afr_changelog_resume_t changelog_resume);
 
-afr_fd_ctx_t *
-__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+
+int
+__afr_txn_write_fop (call_frame_t *frame, xlator_t *this)
 {
-        uint64_t       ctx = 0;
-        int            ret = 0;
-        afr_fd_ctx_t  *fd_ctx = NULL;
-        int            i = 0;
+        afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
 
+        local = frame->local;
         priv = this->private;
 
-        ret = __fd_ctx_get (fd, this, &ctx);
-
-        if (ret < 0 && fd_is_anonymous (fd)) {
-                ret = __afr_fd_ctx_set (this, fd);
-                if (ret < 0)
-                        goto out;
+        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
 
-                ret = __fd_ctx_get (fd, this, &ctx);
-                if (ret < 0)
-                        goto out;
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-                for (i = 0; i < priv->child_count; i++)
-                        fd_ctx->opened_on[i] = AFR_FD_OPENED;
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
         }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-out:
-        return fd_ctx;
-}
-
+        local->call_count = call_count;
 
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this)
-{
-        afr_fd_ctx_t  *fd_ctx = NULL;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+			local->transaction.wind (frame, this, i);
 
-        LOCK(&fd->lock);
-        {
-                fd_ctx = __afr_fd_ctx_get (fd, this);
+                        if (!--call_count)
+                                break;
+                }
         }
-        UNLOCK(&fd->lock);
 
-        return fd_ctx;
+        return 0;
 }
 
 
-static void
-afr_pid_save (call_frame_t *frame)
+int
+__afr_txn_write_done (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = NULL;
+        afr_local_t *local = NULL;
 
         local = frame->local;
 
-        local->saved_pid = frame->root->pid;
-}
-
+        local->transaction.unwind (frame, this);
 
-static void
-afr_pid_restore (call_frame_t *frame)
-{
-        afr_local_t * local = NULL;
+        AFR_STACK_DESTROY (frame);
 
-        local = frame->local;
-
-        frame->root->pid = local->saved_pid;
+        return 0;
 }
 
 
-static void
-__mark_all_pending (int32_t *pending[], int child_count,
-                    afr_transaction_type type)
+call_frame_t*
+afr_transaction_detach_fop_frame (call_frame_t *frame)
 {
-        int i = 0;
-        int j = 0;
-
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-                pending[i][j] = hton32 (1);
-        }
-}
-
+        afr_local_t *   local = NULL;
+        call_frame_t   *fop_frame = NULL;
 
-static void
-__mark_child_dead (int32_t *pending[], int child_count, int child,
-                   afr_transaction_type type)
-{
-        int j = 0;
+        local = frame->local;
 
-        j = afr_index_for_transaction_type (type);
+        LOCK (&frame->lock);
+        {
+                fop_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
 
-        pending[child][j] = 0;
+        return fop_frame;
 }
 
 
 static void
-__mark_pre_op_done_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
+afr_save_lk_owner (call_frame_t *frame)
 {
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+        afr_local_t * local = NULL;
 
         local = frame->local;
 
-        if (!local->fd)
-                return;
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        if (!fd_ctx)
-                goto out;
-
-        LOCK (&local->fd->lock);
-        {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]++;
-        }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
+        local->saved_lk_owner = frame->root->lk_owner;
 }
 
 
 static void
-__mark_pre_op_undone_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
+afr_restore_lk_owner (call_frame_t *frame)
 {
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+        afr_local_t * local = NULL;
 
         local = frame->local;
 
-        if (!local->fd)
-                return;
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        if (!fd_ctx)
-                goto out;
-
-        LOCK (&local->fd->lock);
-        {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]--;
-        }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
+        frame->root->lk_owner = local->saved_lk_owner;
 }
 
-
-static void
-__mark_down_children (int32_t *pending[], int child_count,
-                      unsigned char *child_up, afr_transaction_type type)
+void
+__mark_all_success (call_frame_t *frame, xlator_t *this)
 {
-        int i = 0;
-        int j = 0;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	int i;
 
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
+	local = frame->local;
+	priv = this->private;
 
-                if (!child_up[i])
-                        pending[i][j] = 0;
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		local->transaction.failed_subvols[i] = 0;
+	}
 }
 
 
-static void
-__mark_all_success (int32_t *pending[], int child_count,
-                    afr_transaction_type type)
+int
+afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
 {
-        int i;
-        int j;
+        afr_local_t     *local = NULL;
+        fd_t            *fd   = NULL;
 
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-                pending[i][j] = hton32 (-1);
-        }
+        local = frame->local;
+        fd    = local->fd;
+
+        /*  Perform fops with the lk-owner from top xlator.
+         *  Eg: lk-owner of posix-lk and flush should be same,
+         *  flush cant clear the  posix-lks without that lk-owner.
+         */
+        afr_save_lk_owner (frame);
+        frame->root->lk_owner =
+                local->transaction.main_frame->root->lk_owner;
+
+	if (local->pre_op_compat)
+		/* old mode, pre-op was done as afr_changelog_do()
+		   just now, before OP */
+		afr_changelog_pre_op_update (frame, this);
+
+        /* The wake up needs to happen independent of
+           what type of fop arrives here. If it was
+           a write, then it has already inherited the
+           lock and changelog. If it was not a write,
+           then the presumption of the optimization (of
+           optimizing for successive write operations)
+           fails.
+        */
+        if (fd)
+                afr_delayed_changelog_wake_up (this, fd);
+        local->transaction.fop (frame, this);
+
+	return 0;
 }
 
 
@@ -275,64 +233,31 @@ __fop_changelog_needed (call_frame_t *frame, xlator_t *this)
         return op_ret;
 }
 
+
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending)
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int **pending)
 {
         int i = 0;
         int ret = 0;
+	int pending_zero[AFR_NUM_CHANGE_LOGS] = {0, };
 
         for (i = 0; i < priv->child_count; i++) {
+		if (!memcmp (pending_zero, pending[i], sizeof (pending_zero)))
+			/* don't set xattrs for non-pending servers */
+			continue;
+
                 ret = dict_set_static_bin (xattr, priv->pending_key[i],
-                                           pending[i],
-                                AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+					   pending[i],
+					   AFR_NUM_CHANGE_LOGS * sizeof (int));
                 /* 3 = data+metadata+entry */
 
-                if (ret < 0)
-                        goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-static int
-afr_set_piggyback_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
-                        afr_transaction_type type)
-{
-        int i = 0;
-        int ret = 0;
-        int *arr = NULL;
-        int index = 0;
-        size_t pending_xattr_size = 3 * sizeof (int32_t);
-        /* 3 = data+metadata+entry */
-
-        index = afr_index_for_transaction_type (type);
-
-        for (i = 0; i < priv->child_count; i++) {
-                arr = GF_CALLOC (1, pending_xattr_size,
-                                 gf_afr_mt_char);
-                if (!arr) {
-                        ret = -1;
-                        goto out;
-                }
-
-                memcpy (arr, pending[i], pending_xattr_size);
-
-                arr[index] = hton32 (ntoh32(arr[index]) + 1);
-
-                ret = dict_set_bin (xattr, priv->pending_key[i],
-                                    arr, pending_xattr_size);
-
-                if (ret < 0)
-                        goto out;
+                if (ret)
+                        break;
         }
 
-out:
         return ret;
 }
 
-
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 {
@@ -358,416 +283,532 @@ afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 
 /* {{{ pending */
 
-int32_t
-afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xattr)
+
+int
+afr_changelog_post_op_done (call_frame_t *frame, xlator_t *this)
 {
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
         afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int                  call_count = -1;
 
-        priv     = this->private;
-        local    = frame->local;
+	local = frame->local;
+	priv = this->private;
         int_lock = &local->internal_lock;
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+	if (local->transaction.resume_stub) {
+		call_resume (local->transaction.resume_stub);
+		local->transaction.resume_stub = NULL;
+	}
 
-        if (call_count == 0) {
-                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                        local->transaction.done (frame, this);
-                } else {
-                        int_lock->lock_cbk = local->transaction.done;
-                        afr_unlock (frame, this);
-                }
-        }
+	if (afr_lock_server_count (priv, local->transaction.type) == 0) {
+		local->transaction.done (frame, this);
+	} else {
+		int_lock->lock_cbk = local->transaction.done;
+		afr_unlock (frame, this);
+	}
 
-        return 0;
+	return 0;
 }
 
 
-void
-afr_transaction_rm_stale_children (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode, afr_transaction_type type)
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom)
 {
-        int             i = -1;
-        int             count = 0;
-        int             read_child = -1;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local = NULL;
-        int             **pending = NULL;
-        int             idx = 0;
-        int32_t         *stale_children = NULL;
-        int32_t         *fresh_children = NULL;
-        gf_boolean_t    rm_stale_children = _gf_false;
-
-        idx = afr_index_for_transaction_type (type);
+        afr_inodelk_t *inodelk = NULL;
+        int           i = 0;
 
-        priv = this->private;
-        local = frame->local;
-        pending = local->pending;
-
-        if (local->op_ret < 0)
-                goto out;
-        fresh_children = local->fresh_children;
-        read_child = afr_inode_get_read_ctx (this, inode, fresh_children);
-        if (read_child < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Possible split-brain "
-                        "for %s", uuid_utoa (inode->gfid));
-                goto out;
+        for (i = 0; int_lock->inodelk[i].domain; i++) {
+                inodelk = &int_lock->inodelk[i];
+                if (strcmp (dom, inodelk->domain) == 0)
+                        return inodelk;
         }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!afr_is_child_present (fresh_children,
-                                           priv->child_count, i))
-                        continue;
-                if (pending[i][idx])
-                        continue;
-                /* child is down or op failed on it */
-                if (!stale_children)
-                        stale_children = afr_children_create (priv->child_count);
-                if (!stale_children)
-                        goto out;
-
-                rm_stale_children = _gf_true;
-                stale_children[count++] = i;
-                gf_log (this->name, GF_LOG_DEBUG, "Removing stale child "
-                        "%d for %s", i, uuid_utoa (inode->gfid));
-        }
-
-        if (!rm_stale_children)
-                goto out;
-
-        afr_inode_rm_stale_children (this, inode, stale_children);
-out:
-        if (stale_children)
-                GF_FREE (stale_children);
-        return;
+        return NULL;
 }
 
 unsigned char*
 afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
 {
         unsigned char *locked_nodes = NULL;
+        afr_inodelk_t *inodelk = NULL;
         switch (type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                locked_nodes = int_lock->inode_locked_nodes;
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                locked_nodes = inodelk->locked_nodes;
         break;
 
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                locked_nodes = int_lock->entry_locked_nodes;
+                /*Because same set of subvols participate in all lockee
+                 * entities*/
+                locked_nodes = int_lock->lockee[0].locked_nodes;
         break;
         }
         return locked_nodes;
 }
 
+
 int
-afr_changelog_pre_op_call_count (afr_transaction_type type,
-                                 afr_internal_lock_t *int_lock,
-                                 unsigned int child_count)
+afr_changelog_call_count (afr_transaction_type type,
+			  unsigned char *pre_op_subvols,
+			  unsigned int child_count)
 {
-        int           call_count = 0;
-        unsigned char *locked_nodes = NULL;
+        int call_count = 0;
 
-        locked_nodes = afr_locked_nodes_get (type, int_lock);
-        GF_ASSERT (locked_nodes);
+	call_count = AFR_COUNT(pre_op_subvols, child_count);
 
-        call_count = afr_locked_children_count (locked_nodes, child_count);
         if (type == AFR_ENTRY_RENAME_TRANSACTION)
                 call_count *= 2;
 
         return call_count;
 }
 
-int
-afr_changelog_post_op_call_count (afr_transaction_type type,
-                                  unsigned char *pre_op,
-                                  unsigned int child_count)
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
 {
-        int           call_count = 0;
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int i = 0;
 
-        call_count = afr_pre_op_done_children_count (pre_op, child_count);
-        if (type == AFR_ENTRY_RENAME_TRANSACTION)
-                call_count *= 2;
+        local = frame->local;
+	priv = this->private;
 
-        return call_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.failed_subvols[i])
+                        return _gf_false;
+        }
+
+        return _gf_true;
 }
 
+
+void
+afr_handle_symmetric_errors (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int op_errno = 0;
+	int i_errno = 0;
+	gf_boolean_t matching_errors = _gf_true;
+	int i = 0;
+
+	priv = this->private;
+	local = frame->local;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret != -1) {
+			/* Operation succeeded on at least on subvol,
+			   so it is not a failed-everywhere situation.
+			*/
+			matching_errors = _gf_false;
+			break;
+		}
+		i_errno = local->replies[i].op_errno;
+
+		if (i_errno == ENOTCONN) {
+			/* ENOTCONN is not a symmetric error. We do not
+			   know if the operation was performed on the
+			   backend or not.
+			*/
+			matching_errors = _gf_false;
+			break;
+		}
+
+		if (!op_errno) {
+			op_errno = i_errno;
+		} else if (op_errno != i_errno) {
+			/* Mismatching op_errno's */
+			matching_errors = _gf_false;
+			break;
+		}
+	}
+
+	if (matching_errors)
+		__mark_all_success (frame, this);
+}
+
+
 int
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
 {
         afr_private_t * priv = this->private;
-        afr_internal_lock_t *int_lock = NULL;
-        int ret        = 0;
         int i          = 0;
-        int call_count = 0;
-
+	int ret = 0;
+	int idx = 0;
         afr_local_t *  local = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        dict_t        **xattr = NULL;
-        int            piggyback = 0;
-        int            index = 0;
+        dict_t        *xattr = NULL;
         int            nothing_failed = 1;
+	gf_boolean_t   need_undirty = _gf_false;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        __mark_down_children (local->pending, priv->child_count,
-                              local->child_up, local->transaction.type);
-
-        if (local->fd)
-                afr_transaction_rm_stale_children (frame, this,
-                                                   local->fd->inode,
-                                                   local->transaction.type);
-
-        xattr = alloca (priv->child_count * sizeof (*xattr));
-        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
-        for (i = 0; i < priv->child_count; i++) {
-                xattr[i] = dict_new ();
-        }
+        local = frame->local;
+	idx = afr_index_for_transaction_type (local->transaction.type);
 
-        call_count = afr_changelog_post_op_call_count (local->transaction.type,
-                                                       local->transaction.pre_op,
-                                                       priv->child_count);
-        local->call_count = call_count;
+        nothing_failed = afr_txn_nothing_failed (frame, this);
 
-        if (local->fd)
-                fdctx = afr_fd_ctx_get (local->fd, this);
+	if (afr_changelog_pre_op_uninherit (frame, this))
+		need_undirty = _gf_false;
+	else
+		need_undirty = _gf_true;
 
-        if (call_count == 0) {
-                /* no child is up */
-                int_lock->lock_cbk = local->transaction.done;
-                afr_unlock (frame, this);
+	if (nothing_failed && !need_undirty) {
+		afr_changelog_post_op_done (frame, this);
                 goto out;
-        }
-
-        /* check if something has failed, to handle piggybacking */
-        nothing_failed = 1;
-        index = afr_index_for_transaction_type (local->transaction.type);
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->pending[i][index] == 0) {
-                        nothing_failed = 0;
-                        break;
-                }
-        }
+	}
+
+	xattr = dict_new ();
+	if (!xattr) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		afr_changelog_post_op_done (frame, this);
+		goto out;
+	}
+
+	if (need_undirty) {
+		local->dirty[idx] = hton32(-1);
+
+		ret = dict_set_static_bin (xattr, AFR_DIRTY, local->dirty,
+					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			afr_changelog_post_op_done (frame, this);
+			goto out;
+		}
+
+	}
+
+	if (!nothing_failed) {
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.failed_subvols[i])
+				local->pending[i][idx] = hton32(1);
+		}
+		ret = afr_set_pending_dict (priv, xattr, local->pending);
+		if (ret < 0) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			afr_changelog_post_op_done (frame, this);
+			goto out;
+		}
+
+	}
+
+	afr_changelog_do (frame, this, xattr, afr_changelog_post_op_done);
+out:
+	if (xattr)
+                dict_unref (xattr);
 
-        index = afr_index_for_transaction_type (local->transaction.type);
-        if (local->optimistic_change_log &&
-            local->transaction.type != AFR_DATA_TRANSACTION) {
-                /* if nothing_failed, then local->pending[..] == {0 .. 0} */
-                for (i = 0; i < priv->child_count; i++)
-                        local->pending[i][index]++;
-        }
+        return 0;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i])
-                        continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
+gf_boolean_t
+afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int type = 0;
+
+	local = frame->local;
+	priv = this->private;
+	fd = local->fd;
+
+	type = afr_index_for_transaction_type (local->transaction.type);
+	if (type != AFR_DATA_TRANSACTION)
+		return !local->transaction.dirtied;
+
+	if (!fd)
+		return !local->transaction.dirtied;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
+
+	if (local->transaction.no_uninherit)
+		return _gf_false;
+
+	/* This function must be idempotent. So check if we
+	   were called before and return the same answer again.
+
+	   It is important to keep this function idempotent for
+	   the call in afr_changelog_post_op_safe() to not have
+	   side effects on the call from afr_changelog_post_op_now()
+	*/
+	if (local->transaction.uninherit_done)
+		return local->transaction.uninherit_value;
+
+	LOCK(&fd->lock);
+	{
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.pre_op[i] !=
+			    fd_ctx->pre_op_done[type][i]) {
+				ret = !local->transaction.dirtied;
+				goto unlock;
+			}
+		}
+
+		if (fd_ctx->inherited[type]) {
+			ret = _gf_true;
+			fd_ctx->inherited[type]--;
+		} else if (fd_ctx->on_disk[type]) {
+			ret = _gf_false;
+			fd_ctx->on_disk[type]--;
+		} else {
+			/* ASSERT */
+			ret = _gf_false;
+		}
+
+		if (!fd_ctx->inherited[type] && !fd_ctx->on_disk[type]) {
+			for (i = 0; i < priv->child_count; i++)
+				fd_ctx->pre_op_done[type][i] = 0;
+		}
+	}
+unlock:
+	UNLOCK(&fd->lock);
+
+	local->transaction.uninherit_done = _gf_true;
+	local->transaction.uninherit_value = ret;
+
+	return ret;
+}
 
 
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                {
-                        if (!fdctx) {
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                                break;
-                        }
+gf_boolean_t
+afr_changelog_pre_op_inherit (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int type = 0;
+
+	local = frame->local;
+	priv = this->private;
+	fd = local->fd;
+
+	if (local->transaction.type != AFR_DATA_TRANSACTION)
+		return _gf_false;
+
+	type = afr_index_for_transaction_type (local->transaction.type);
+
+	if (!fd)
+		return _gf_false;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
+
+	LOCK(&fd->lock);
+	{
+		if (!fd_ctx->on_disk[type]) {
+			/* nothing to inherit yet */
+			ret = _gf_false;
+			goto unlock;
+		}
+
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.pre_op[i] !=
+			    fd_ctx->pre_op_done[type][i]) {
+				/* either inherit exactly, or don't */
+				ret = _gf_false;
+				goto unlock;
+			}
+		}
+
+		fd_ctx->inherited[type]++;
+
+		ret = _gf_true;
+
+		local->transaction.inherited = _gf_true;
+	}
+unlock:
+	UNLOCK(&fd->lock);
+
+	return ret;
+}
 
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_piggyback[i]) {
-                                        fdctx->pre_op_piggyback[i]--;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
 
-                        if (piggyback && !nothing_failed)
-                                ret = afr_set_piggyback_dict (priv, xattr[i],
-                                                              local->pending,
-                                                              local->transaction.type);
+gf_boolean_t
+afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	int type = 0;
+
+	local = frame->local;
+	priv = this->private;
+	fd = local->fd;
+
+	if (!fd)
+		return _gf_false;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
+
+	if (local->transaction.inherited)
+		/* was already inherited in afr_changelog_pre_op */
+		return _gf_false;
+
+	if (!local->transaction.dirtied)
+		return _gf_false;
+
+        if (!afr_txn_nothing_failed (frame, this))
+		return _gf_false;
+
+	type = afr_index_for_transaction_type (local->transaction.type);
+
+	ret = _gf_false;
+
+	LOCK(&fd->lock);
+	{
+		if (!fd_ctx->on_disk[type]) {
+			for (i = 0; i < priv->child_count; i++)
+				fd_ctx->pre_op_done[type][i] =
+					local->transaction.pre_op[i];
+		} else {
+			for (i = 0; i < priv->child_count; i++)
+				if (fd_ctx->pre_op_done[type][i] !=
+				    local->transaction.pre_op[i]) {
+					local->transaction.no_uninherit = 1;
+					goto unlock;
+				}
+		}
+		fd_ctx->on_disk[type]++;
+
+		ret = _gf_true;
+	}
+unlock:
+	UNLOCK(&fd->lock);
+
+	return ret;
+}
 
-                        if (nothing_failed && piggyback) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
-                        } else {
-                                __mark_pre_op_undone_on_fd (frame, this, i);
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_post_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        }
-                }
-                break;
-                case AFR_METADATA_TRANSACTION:
-                {
-                        if (nothing_failed) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
-                                break;
-                        }
 
-                        if (local->fd)
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->fxattrop,
-                                            local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        else
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
+int
+afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
 
-                case AFR_ENTRY_RENAME_TRANSACTION:
-                {
-                        if (nothing_failed) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
-                        } else {
-                                STACK_WIND_COOKIE (frame, afr_changelog_post_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        }
-                        call_count--;
-                }
+        local = frame->local;
 
-                /*
-                  set it again because previous stack_wind
-                  might have already returned (think of case
-                  where subvolume is posix) and would have
-                  used the dict as placeholder for return
-                  value
-                */
+	if (op_ret == -1)
+		afr_transaction_fop_failed (frame, this, (long) cookie);
 
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
+	call_count = afr_frame_return (frame);
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
+        if (call_count == 0)
+		local->transaction.changelog_resume (frame, this);
 
-                /* fall through */
+        return 0;
+}
 
-                case AFR_ENTRY_TRANSACTION:
-                {
-                        if (nothing_failed) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i]);
-                                break;
-                        }
 
-                        if (local->fd)
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->fxattrop,
-                                            local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        else
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->transaction.parent_loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
-                }
+int
+afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
+		  afr_changelog_resume_t changelog_resume)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int call_count = 0;
 
-                if (!--call_count)
-                        break;
-        }
+	local = frame->local;
+	priv = this->private;
 
-out:
-        for (i = 0; i < priv->child_count; i++) {
-                dict_unref (xattr[i]);
-        }
+        call_count = afr_changelog_call_count (local->transaction.type,
+					       local->transaction.pre_op,
+					       priv->child_count);
 
-        return 0;
-}
+	if (call_count == 0) {
+		changelog_resume (frame, this);
+		return 0;
+	}
 
+	local->call_count = call_count;
 
-int32_t
-afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = this->private;
-        int call_count  = -1;
-        int child_index = (long) cookie;
+	local->transaction.changelog_resume = changelog_resume;
 
-        local = frame->local;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
 
-        LOCK (&frame->lock);
-        {
-                switch (op_ret) {
-                case 0:
-                        __mark_pre_op_done_on_fd (frame, this, child_index);
-                        //fallthrough we need to mark the pre_op
-                case 1:
-                        local->transaction.pre_op[child_index] = 1;
-                        /* special op_ret for piggyback */
-                        break;
-                case -1:
-                        if (op_errno == ENOTSUP) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "xattrop not supported by %s",
-                                        priv->children[child_index]->name);
-                                local->op_ret = -1;
-
-                        } else if (!child_went_down (op_ret, op_errno)) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "xattrop failed on child %s: %s",
-                                        priv->children[child_index]->name,
-                                        strerror (op_errno));
+                switch (local->transaction.type) {
+                case AFR_DATA_TRANSACTION:
+                case AFR_METADATA_TRANSACTION:
+                        if (!local->fd) {
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->xattrop,
+						   &local->loc,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
+                        } else {
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->fxattrop,
+						   local->fd,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
                         }
-                        local->op_errno = op_errno;
-                        break;
-                }
+			break;
+                case AFR_ENTRY_RENAME_TRANSACTION:
 
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+			STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+					   (void *) (long) i,
+					   priv->children[i],
+					   priv->children[i]->fops->xattrop,
+					   &local->transaction.new_parent_loc,
+					   GF_XATTROP_ADD_ARRAY, xattr,
+					   NULL);
+                        call_count--;
 
-        if (call_count == 0) {
-                if ((local->op_ret == -1) &&
-                    (local->op_errno == ENOTSUP)) {
-                        local->transaction.resume (frame, this);
-                } else {
-                        __mark_all_success (local->pending, priv->child_count,
-                                            local->transaction.type);
+                /* fall through */
 
-                        afr_pid_restore (frame);
+                case AFR_ENTRY_TRANSACTION:
+                        if (local->fd)
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->fxattrop,
+						   local->fd,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
+                        else
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->xattrop,
+						   &local->transaction.parent_loc,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
+			break;
+		}
 
-                        local->transaction.fop (frame, this);
-                }
+                if (!--call_count)
+                        break;
         }
 
-        return 0;
+	return 0;
 }
 
+
 int
 afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
 {
@@ -775,191 +816,122 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
         int i = 0;
         int ret = 0;
         int call_count = 0;
-        dict_t **xattr = NULL;
-        afr_fd_ctx_t *fdctx = NULL;
+	int op_errno = 0;
         afr_local_t *local = NULL;
-        int          piggyback = 0;
         afr_internal_lock_t *int_lock = NULL;
         unsigned char       *locked_nodes = NULL;
+	unsigned char       *pending_subvols = NULL;
+	int idx = -1;
+	gf_boolean_t pre_nop = _gf_true;
+	dict_t *xdata_req = NULL;
 
         local = frame->local;
         int_lock = &local->internal_lock;
-
-        xattr = alloca (priv->child_count * sizeof (*xattr));
-        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
-
-        for (i = 0; i < priv->child_count; i++) {
-                xattr[i] = dict_new ();
-        }
-
-        call_count = afr_changelog_pre_op_call_count (local->transaction.type,
-                                                      int_lock,
-                                                      priv->child_count);
-        if (call_count == 0) {
-                local->internal_lock.lock_cbk =
-                        local->transaction.done;
-                afr_unlock (frame, this);
-                goto out;
-        }
-
-        local->call_count = call_count;
-
-        __mark_all_pending (local->pending, priv->child_count,
-                            local->transaction.type);
-
-        if (local->fd)
-                fdctx = afr_fd_ctx_get (local->fd, this);
+	idx = afr_index_for_transaction_type (local->transaction.type);
 
         locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!locked_nodes[i])
-                        continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
-
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
-
-
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                {
-                        if (!fdctx) {
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                                break;
-                        }
-
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_done[i]) {
-                                        fdctx->pre_op_piggyback[i]++;
-                                        piggyback = 1;
-                                        fdctx->hit++;
-                                } else {
-                                        fdctx->miss++;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        if (piggyback)
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
-                case AFR_METADATA_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
-                                break;
-                        }
-
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
-
-                case AFR_ENTRY_RENAME_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
-                        } else {
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        }
-
-                        call_count--;
-                }
 
+	pending_subvols = alloca0 (priv->child_count);
 
-                /*
-                  set it again because previous stack_wind
-                  might have already returned (think of case
-                  where subvolume is posix) and would have
-                  used the dict as placeholder for return
-                  value
-                */
+	for (i = 0; i < priv->child_count; i++) {
+		if (locked_nodes[i]) {
+			local->transaction.pre_op[i] = 1;
+			call_count++;
+		} else {
+			pending_subvols[i] = 1;
+		}
+	}
 
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
+	/* TBD: quorum check w/ call_count */
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
-
-                /* fall through */
+        if (call_count == 0) {
+		op_errno = ENOTCONN;
+		goto err;
+	}
+
+	xdata_req = dict_new();
+	if (!xdata_req) {
+		op_errno = ENOMEM;
+		goto err;
+	}
+
+	pre_nop = _gf_true;
+
+	if (afr_changelog_pre_op_inherit (frame, this))
+		goto next;
+
+	if (call_count < priv->child_count) {
+		/* For subvols we are not performing operation on,
+		   mark them as pending up-front along with the FOP
+		   so that we can safely defer unmarking dirty until
+		   later.
+		*/
+		for (i = 0; i < priv->child_count; i++) {
+			if (pending_subvols[i])
+				local->pending[i][idx] = hton32(1);
+		}
+		ret = afr_set_pending_dict (priv, xdata_req,
+					    local->pending);
+		if (ret < 0) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		pre_nop = _gf_false;
+	}
+
+	if (call_count > 1 &&
+	    (local->transaction.type == AFR_DATA_TRANSACTION ||
+	     !local->optimistic_change_log)) {
+
+		/* If we are performing change on only one subvol, no
+		   need to mark dirty, because we are setting the pending
+		   counts already anyways
+		*/
+		local->dirty[idx] = hton32(1);
+
+		ret = dict_set_static_bin (xdata_req, AFR_DIRTY, local->dirty,
+					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+
+		pre_nop = _gf_false;
+		local->transaction.dirtied = 1;
+	}
+
+	if (pre_nop)
+		goto next;
+
+	if (!local->pre_op_compat) {
+		dict_copy (xdata_req, local->xdata_req);
+		goto next;
+	}
+
+	afr_changelog_do (frame, this, xdata_req, afr_transaction_perform_fop);
+
+	if (xdata_req)
+		dict_unref (xdata_req);
+
+	return 0;
+next:
+	afr_transaction_perform_fop (frame, this);
+
+	if (xdata_req)
+		dict_unref (xdata_req);
 
-                case AFR_ENTRY_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
-                                break;
-                        }
+        return 0;
+err:
+	local->internal_lock.lock_cbk = local->transaction.done;
+	local->op_ret = -1;
+	local->op_errno = op_errno;
 
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
-                }
+	afr_unlock (frame, this);
 
-                if (!--call_count)
-                        break;
-        }
-out:
-        for (i = 0; i < priv->child_count; i++) {
-                dict_unref (xattr[i]);
-        }
+	if (xdata_req)
+		dict_unref (xdata_req);
 
-        return 0;
+	return 0;
 }
 
 
@@ -1108,12 +1080,14 @@ int
 afr_set_transaction_flock (afr_local_t *local)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
 
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        int_lock->lk_flock.l_len   = local->transaction.len;
-        int_lock->lk_flock.l_start = local->transaction.start;
-        int_lock->lk_flock.l_type  = F_WRLCK;
+        inodelk->flock.l_len   = local->transaction.len;
+        inodelk->flock.l_start = local->transaction.start;
+        inodelk->flock.l_type  = F_WRLCK;
 
         return 0;
 }
@@ -1128,6 +1102,7 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
+        int_lock->domain = this->name;
 
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
@@ -1141,8 +1116,8 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 
         case AFR_ENTRY_RENAME_TRANSACTION:
 
-                int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-                afr_blocking_lock (frame, this);
+                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
+                afr_nonblocking_entrylk (frame, this);
                 break;
 
         case AFR_ENTRY_TRANSACTION:
@@ -1164,12 +1139,6 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 int
 afr_lock (call_frame_t *frame, xlator_t *this)
 {
-        afr_pid_save (frame);
-
-        frame->root->pid = (long) frame->root;
-
-        afr_set_lk_owner (frame, this);
-
         afr_set_lock_number (frame, this);
 
         return afr_lock_rec (frame, this);
@@ -1181,47 +1150,427 @@ afr_lock (call_frame_t *frame, xlator_t *this)
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-
         if (__fop_changelog_needed (frame, this)) {
                 afr_changelog_pre_op (frame, this);
         } else {
-                __mark_all_success (local->pending, priv->child_count,
-                                    local->transaction.type);
+                afr_transaction_perform_fop (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+        /* call this function from any of the related optimizations
+           which benefit from delaying post op are enabled, namely:
+
+           - changelog piggybacking
+           - eager locking
+        */
+
+        priv = this->private;
+        if (!priv)
+                return;
+
+        if (!priv->post_op_delay_secs)
+                return;
+
+        local = frame->local;
+        if (!local->transaction.eager_lock_on)
+                return;
+
+        if (!local)
+                return;
+
+        if (!local->fd)
+                return;
+
+        if (local->op == GF_FOP_WRITE)
+                local->delayed_post_op = _gf_true;
+}
+
+gf_boolean_t
+afr_are_multiple_fds_opened (fd_t *fd, xlator_t *this)
+{
+        afr_fd_ctx_t *fd_ctx = NULL;
+
+        if (!fd) {
+                /* If false is returned, it may keep on taking eager-lock
+                 * which may lead to starvation, so return true to avoid that.
+                 */
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid fd");
+                return _gf_true;
+        }
+        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+         * is taken mount2 opened the same file, it won't be able to
+         * perform any data operations until mount1 releases eager-lock.
+         * To avoid such scenario do not enable eager-lock for this transaction
+         * if open-fd-count is > 1
+         */
+
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
+                return _gf_true;
+
+        if (fd_ctx->open_fd_count > 1)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+
+gf_boolean_t
+is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t      *local = NULL;
+        gf_boolean_t      res = _gf_false;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        if (!local->delayed_post_op)
+                goto out;
+
+        //Mark pending changelog ASAP
+        if (!afr_txn_nothing_failed (frame, this))
+                goto out;
+
+        if (local->fd && afr_are_multiple_fds_opened (local->fd, this))
+                goto out;
+
+        res = _gf_true;
+out:
+        return res;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub);
+
+void
+afr_delayed_changelog_wake_up_cbk (void *data)
+{
+        fd_t           *fd = NULL;
+
+        fd = data;
+
+        afr_delayed_changelog_wake_up (THIS, fd);
+}
+
+
+/* SET operation */
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+
+        fdctx = afr_fd_ctx_get (fd, this);
+
+        LOCK(&fd->lock);
+        {
+                fdctx->witnessed_unstable_write = _gf_true;
+        }
+        UNLOCK(&fd->lock);
+
+        return 0;
+}
+
+/* TEST and CLEAR operation */
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t witness = _gf_false;
+
+        fdctx = afr_fd_ctx_get (fd, this);
+        if (!fdctx)
+                return _gf_true;
+
+        LOCK(&fd->lock);
+        {
+                if (fdctx->witnessed_unstable_write) {
+                        witness = _gf_true;
+                        fdctx->witnessed_unstable_write = _gf_false;
+                }
+        }
+        UNLOCK (&fd->lock);
+
+        return witness;
+}
+
 
-                afr_pid_restore (frame);
+int
+afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *pre,
+                         struct iatt *post, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
+        afr_local_t *local = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (op_ret != 0) {
+                /* Failure of fsync() is as good as failure of previous
+                   write(). So treat it like one.
+		*/
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fsync(%s) failed on subvolume %s. Transaction was %s",
+                        uuid_utoa (local->fd->inode->gfid),
+                        priv->children[child_index]->name,
+                        gf_fop_list[local->op]);
+
+                afr_transaction_fop_failed (frame, this, child_index);
+	}
+
+	call_count = afr_frame_return (frame);
+
+	if (call_count == 0)
+		afr_changelog_post_op_now (frame, this);
+
+	return 0;
+}
+
+
+int
+afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        int i = 0;
+        int call_count = 0;
+        afr_private_t *priv = NULL;
+        dict_t *xdata = NULL;
+        GF_UNUSED int ret = -1;
 
-                local->transaction.fop (frame, this);
+        local = frame->local;
+        priv = this->private;
+
+        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+
+        if (!call_count) {
+                /* will go straight to unlock */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
         }
 
+        local->call_count = call_count;
+
+        xdata = dict_new();
+        if (xdata)
+                ret = dict_set_int32 (xdata, "batch-fsync", 1);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
+				   (void *) (long) i, priv->children[i],
+				   priv->children[i]->fops->fsync, local->fd,
+				   1, xdata);
+                if (!--call_count)
+                        break;
+        }
+
+        if (xdata)
+                dict_unref (xdata);
+
         return 0;
 }
 
 
 int
+afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+	local = frame->local;
+        priv = this->private;
+
+        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        if (afr_changelog_pre_op_uninherit (frame, this) &&
+	    afr_txn_nothing_failed (frame, this)) {
+                /* just detected that this post-op is about to
+                   be optimized away as a new write() has
+                   already piggybacked on this frame's changelog.
+                   */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Calling afr_changelog_post_op_now() now will result in
+           issuing ->[f]xattrop().
+
+           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+           responsible operation that what it might appear on the surface.
+
+           The changelog of a file (in the xattr of the file on the server)
+           stores information (pending count) about the state of the file
+           on the OTHER server. This changelog is blindly trusted, and must
+           therefore be updated in such a way it remains trustworthy. This
+           implies that decrementing the pending count (essentially "clearing
+           the dirty flag") must be done STRICTLY after we are sure that the
+           operation on the other server has reached stable storage.
+
+           While the backend filesystem on that server will eventually flush
+           it to stable storage, we (being in userspace) have no mechanism
+           to get notified when the write became "stable".
+
+           This means we need take matter into our own hands and issue an
+           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+           and get an acknowledgement for it. And we need to wait for the
+           fsync() acknowledgement before initiating the hard POST-OP.
+
+           However if the FD itself was opened in O_SYNC or O_DSYNC then
+           we are already guaranteed that the writes were made stable as
+           part of the FOP itself. The same holds true for NFS stable
+           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+           flag set in the writev() @flags param. For all other write types,
+           mark a flag in the fdctx whenever an unstable write is witnessed.
+           */
+
+        if (!afr_fd_has_witnessed_unstable_write (this, local->fd)) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Check whether users want durability and perform fsync/post-op
+         * accordingly.
+         */
+        if (priv->ensure_durability) {
+                /* Time to fsync() */
+                afr_changelog_fsync (frame, this);
+        } else {
+                afr_changelog_post_op_now (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub)
+{
+	afr_fd_ctx_t      *fd_ctx = NULL;
+	call_frame_t      *prev_frame = NULL;
+	struct timespec    delta = {0, };
+	afr_private_t     *priv = NULL;
+	afr_local_t       *local = NULL;
+
+	priv = this->private;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+                goto out;
+
+	delta.tv_sec = priv->post_op_delay_secs;
+	delta.tv_nsec = 0;
+
+	pthread_mutex_lock (&fd_ctx->delay_lock);
+	{
+		prev_frame = fd_ctx->delay_frame;
+		fd_ctx->delay_frame = NULL;
+		if (fd_ctx->delay_timer)
+			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
+		fd_ctx->delay_timer = NULL;
+		if (!frame)
+			goto unlock;
+		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
+							   afr_delayed_changelog_wake_up_cbk,
+							   fd);
+		fd_ctx->delay_frame = frame;
+	}
+unlock:
+	pthread_mutex_unlock (&fd_ctx->delay_lock);
+
+out:
+	if (prev_frame) {
+		local = prev_frame->local;
+		local->transaction.resume_stub = stub;
+		afr_changelog_post_op_now (prev_frame, this);
+	} else if (stub) {
+		call_resume (stub);
+	}
+}
+
+
+void
+afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (is_afr_delayed_changelog_post_op_needed (frame, this))
+                afr_delayed_changelog_post_op (this, frame, local->fd, NULL);
+        else
+                afr_changelog_post_op_safe (frame, this);
+}
+
+
+
+/* Wake up the sleeping/delayed post-op, and also register
+   a stub to have it resumed after this transaction
+   completely finishes.
+
+   The @stub gets saved in @local and gets resumed in
+   afr_local_cleanup()
+   */
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, stub);
+}
+
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, NULL);
+}
+
+
+int
 afr_transaction_resume (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
 
         local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
+
+        if (local->transaction.eager_lock_on) {
+                /* We don't need to retain "local" in the
+                   fd list anymore, writes to all subvols
+                   are finished by now */
+                afr_remove_eager_lock_stub (local);
+        }
+
+        afr_restore_lk_owner (frame);
+
+        afr_handle_symmetric_errors (frame, this);
+
+	if (!local->pre_op_compat)
+		/* new mode, pre-op was done along
+		   with OP */
+		afr_changelog_pre_op_update (frame, this);
 
         if (__fop_changelog_needed (frame, this)) {
                 afr_changelog_post_op (frame, this);
         } else {
-                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                        local->transaction.done (frame, this);
-                } else {
-                        int_lock->lock_cbk = local->transaction.done;
-                        afr_unlock (frame, this);
-                }
+		afr_changelog_post_op_done (frame, this);
         }
 
         return 0;
@@ -1233,16 +1582,96 @@ afr_transaction_resume (call_frame_t *frame, xlator_t *this)
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this, int child_index)
+afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
+                            int child_index)
 {
         afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
 
         local = frame->local;
-        priv  = this->private;
 
-        __mark_child_dead (local->pending, priv->child_count,
-                           child_index, local->transaction.type);
+	local->transaction.failed_subvols[child_index] = 1;
+}
+
+
+
+        static gf_boolean_t
+afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+{
+        uint64_t start1 = local1->transaction.start;
+        uint64_t start2 = local2->transaction.start;
+        uint64_t end1 = 0;
+        uint64_t end2 = 0;
+
+        if (local1->transaction.len)
+                end1 = start1 + local1->transaction.len - 1;
+        else
+                end1 = ULLONG_MAX;
+
+        if (local2->transaction.len)
+                end2 = start2 + local2->transaction.len - 1;
+        else
+                end2 = ULLONG_MAX;
+
+        return ((end1 >= start2) && (end2 >= start1));
+}
+
+void
+afr_transaction_eager_lock_init (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        afr_local_t   *each = NULL;
+
+        priv = this->private;
+
+        if (!local->fd)
+                return;
+
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+
+        if (!priv->eager_lock)
+                return;
+
+        fdctx = afr_fd_ctx_get (local->fd, this);
+        if (!fdctx)
+                return;
+
+        if (afr_are_multiple_fds_opened (local->fd, this))
+                return;
+        /*
+         * Once full file lock is acquired in eager-lock phase, overlapping
+         * writes do not compete for inode-locks, instead are transferred to the
+         * next writes. Because of this overlapping writes are not ordered.
+         * This can cause inconsistencies in replication.
+         * Example:
+         * Two overlapping writes w1, w2 are sent in parallel on same fd
+         * in two threads t1, t2.
+         * Both threads can execute afr_writev_wind in the following manner.
+         * t1 winds w1 on brick-0
+         * t2 winds w2 on brick-0
+         * t2 winds w2 on brick-1
+         * t1 winds w1 on brick-1
+         *
+         * This check makes sure the locks are not transferred for
+         * overlapping writes.
+         */
+        LOCK (&local->fd->lock);
+        {
+                list_for_each_entry (each, &fdctx->eager_locked,
+                                     transaction.eager_locked) {
+                        if (afr_locals_overlap (each, local)) {
+                                local->transaction.eager_lock_on = _gf_false;
+                                goto unlock;
+                        }
+                }
+
+                local->transaction.eager_lock_on = _gf_true;
+                list_add_tail (&local->transaction.eager_locked,
+                               &fdctx->eager_locked);
+        }
+unlock:
+        UNLOCK (&local->fd->lock);
 }
 
 
@@ -1251,20 +1680,43 @@ afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
+        fd_t            *fd   = NULL;
+        int             ret   = -1;
 
         local = frame->local;
         priv  = this->private;
 
-        afr_transaction_local_init (local, this);
-
         local->transaction.resume = afr_transaction_resume;
         local->transaction.type   = type;
 
+        ret = afr_transaction_local_init (local, this);
+        if (ret < 0)
+            goto out;
+
+        afr_transaction_eager_lock_init (local, this);
+
+        if (local->fd && local->transaction.eager_lock_on)
+                afr_set_lk_owner (frame, this, local->fd);
+        else
+                afr_set_lk_owner (frame, this, frame->root);
+
+        if (!local->transaction.eager_lock_on && local->loc.inode) {
+                fd = fd_lookup (local->loc.inode, frame->root->pid);
+                if (fd == NULL)
+                        fd = fd_lookup_anonymous (local->loc.inode);
+
+                if (fd) {
+                        afr_delayed_changelog_wake_up (this, fd);
+                        fd_unref (fd);
+                }
+        }
+
         if (afr_lock_server_count (priv, local->transaction.type) == 0) {
                 afr_internal_lock_finish (frame, this);
         } else {
                 afr_lock (frame, this);
         }
-
-        return 0;
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index f470f2697..77cc8eed0 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -1,25 +1,18 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __TRANSACTION_H__
 #define __TRANSACTION_H__
 
+#include "afr.h"
+
 void
 afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 			    int child_index);
@@ -27,11 +20,34 @@ afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
 
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom);
+
 int32_t
 afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 int
 afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending);
+
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+
+void
+__mark_all_success (call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
+
+int afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		  afr_read_txn_wind_t readfn, afr_transaction_type type);
+
+int afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol);
+
+int __afr_txn_write_fop (call_frame_t *frame, xlator_t *this);
+int __afr_txn_write_done (call_frame_t *frame, xlator_t *this);
+call_frame_t *afr_transaction_detach_fop_frame (call_frame_t *frame);
+
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index ce82aede8..5e12910b7 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -30,11 +21,6 @@
 #endif
 #include "afr-common.c"
 
-#define SHD_INODE_LRU_LIMIT          100
-#define AFR_EH_HEALED_LIMIT          1024
-#define AFR_EH_HEAL_FAIL_LIMIT       1024
-#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
-
 struct volume_options options[];
 
 int32_t
@@ -116,12 +102,21 @@ reconfigure (xlator_t *this, dict_t *options)
 {
         afr_private_t *priv        = NULL;
         xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
         int            ret         = -1;
         int            index       = -1;
         char          *qtype       = NULL;
 
         priv = this->private;
 
+	GF_OPTION_RECONF ("afr-dirty-xattr",
+			  priv->afr_dirty, options, str,
+			  out);
+
+	GF_OPTION_RECONF ("metadata-splitbrain-forced-heal",
+			  priv->metadata_splitbrain_forced_heal, options, bool,
+			  out);
+
         GF_OPTION_RECONF ("background-self-heal-count",
                           priv->background_self_heal_count, options, uint32,
                           out);
@@ -135,9 +130,6 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
                           bool, out);
 
-        GF_OPTION_RECONF ("strict-readdir", priv->strict_readdir, options, bool,
-                          out);
-
         GF_OPTION_RECONF ("data-self-heal-window-size",
                           priv->data_self_heal_window_size, options,
                           uint32, out);
@@ -154,10 +146,11 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("data-self-heal-algorithm",
                           priv->data_self_heal_algorithm, options, str, out);
 
-        GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options, bool, out);
-
         GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
 
+        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
+                          options, uint32, out);
+
         if (read_subvol) {
                 index = xlator_subvolume_index (this, read_subvol);
                 if (index == -1) {
@@ -168,11 +161,43 @@ reconfigure (xlator_t *this, dict_t *options)
                 priv->read_child = index;
         }
 
+        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
+
+        if (read_subvol_index >-1) {
+                index=read_subvol_index;
+                if (index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                index);
+                        goto out;
+                }
+                priv->read_child = index;
+        }
+
+        GF_OPTION_RECONF ("pre-op-compat", priv->pre_op_compat, options, bool, out);
+
+        GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
         GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
         GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
                           uint32, out);
         fix_quorum_options(this,priv,qtype);
 
+	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
+			  uint32, out);
+
+        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
+                          options, size, out);
+        /* Reset this so we re-discover in case the topology changed.  */
+        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
+                          bool, out);
+
+	GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options,
+			  bool, out);
+
+	GF_OPTION_RECONF ("iam-self-heal-daemon", priv->shd.iamshd, options,
+			  bool, out);
+
+        priv->did_discovery = _gf_false;
+
         ret = 0;
 out:
         return ret;
@@ -199,6 +224,7 @@ init (xlator_t *this)
         int            ret         = -1;
         GF_UNUSED int  op_errno    = 0;
         xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
         xlator_t      *fav_child   = NULL;
         char          *qtype       = NULL;
 
@@ -221,18 +247,18 @@ init (xlator_t *this)
 
         priv = this->private;
         LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->read_child_lock);
-        //lock recovery is not done in afr
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->saved_fds);
 
         child_count = xlator_subvolume_count (this);
 
         priv->child_count = child_count;
 
-
         priv->read_child = -1;
 
+	GF_OPTION_INIT ("afr-dirty-xattr", priv->afr_dirty, str, out);
+
+	GF_OPTION_INIT ("metadata-splitbrain-forced-heal",
+			priv->metadata_splitbrain_forced_heal, bool, out);
+
         GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
         if (read_subvol) {
                 priv->read_child = xlator_subvolume_index (this, read_subvol);
@@ -242,6 +268,18 @@ init (xlator_t *this)
                         goto out;
                 }
         }
+        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
+        if (read_subvol_index > -1) {
+                if (read_subvol_index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                read_subvol_index);
+                        goto out;
+                }
+                priv->read_child = read_subvol_index;
+        }
+        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
+
+        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
 
         priv->favorite_child = -1;
         GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
@@ -274,10 +312,6 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
 
-        GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
-
-        GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
-
         GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);
 
         GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool,
@@ -292,12 +326,23 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("entrylk-trace", priv->entrylk_trace, bool, out);
 
-        GF_OPTION_INIT ("strict-readdir", priv->strict_readdir, bool, out);
+        GF_OPTION_INIT ("pre-op-compat", priv->pre_op_compat, bool, out);
 
+        GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
         GF_OPTION_INIT ("quorum-type", qtype, str, out);
         GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
+        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size,
+                        out);
         fix_quorum_options(this,priv,qtype);
 
+	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
+                        out);
+
+	GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
+
+	GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
+
         priv->wait_count = 1;
 
         priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
@@ -338,8 +383,6 @@ init (xlator_t *this)
                                    AFR_XATTR_PREFIX,
                                    trav->xlator->name);
                 if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed to set pending key");
                         ret = -ENOMEM;
                         goto out;
                 }
@@ -348,6 +391,13 @@ init (xlator_t *this)
                 i++;
         }
 
+        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
+                           this->name);
+        if (-1 == ret) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
                                       gf_afr_mt_int32_t);
         if (!priv->last_event) {
@@ -355,6 +405,12 @@ init (xlator_t *this)
                 goto out;
         }
 
+	ret = afr_selfheal_daemon_init (this);
+	if (ret) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
         /* keep more local here as we may need them for self-heal etc */
         this->local_pool = mem_pool_new (afr_local_t, 512);
         if (!this->local_pool) {
@@ -364,56 +420,8 @@ init (xlator_t *this)
                 goto out;
         }
 
-        priv->first_lookup = 1;
         priv->root_inode = NULL;
 
-        if (!priv->shd.iamshd) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = -ENOMEM;
-        priv->shd.pos = GF_CALLOC (sizeof (*priv->shd.pos), child_count,
-                                   gf_afr_mt_brick_pos_t);
-        if (!priv->shd.pos)
-                goto out;
-
-        priv->shd.pending = GF_CALLOC (sizeof (*priv->shd.pending), child_count,
-                                       gf_afr_mt_int32_t);
-        if (!priv->shd.pending)
-                goto out;
-
-        priv->shd.inprogress = GF_CALLOC (sizeof (*priv->shd.inprogress),
-                                          child_count, gf_afr_mt_shd_bool_t);
-        if (!priv->shd.inprogress)
-                goto out;
-        priv->shd.timer = GF_CALLOC (sizeof (*priv->shd.timer), child_count,
-                                     gf_afr_mt_shd_timer_t);
-        if (!priv->shd.timer)
-                goto out;
-
-        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false);
-        if (!priv->shd.healed)
-                goto out;
-
-        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false);
-        if (!priv->shd.heal_failed)
-                goto out;
-
-        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false);
-        if (!priv->shd.split_brain)
-                goto out;
-
-        priv->shd.sh_times = GF_CALLOC (priv->child_count,
-                                        sizeof (*priv->shd.sh_times),
-                                        gf_afr_mt_time_t);
-        if (!priv->shd.sh_times)
-                goto out;
-
-        this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
-        if (!this->itable)
-                goto out;
-
         ret = 0;
 out:
         return ret;
@@ -448,6 +456,9 @@ struct xlator_fops fops = {
         .finodelk    = afr_finodelk,
         .entrylk     = afr_entrylk,
         .fentrylk    = afr_fentrylk,
+	.fallocate   = afr_fallocate,
+	.discard     = afr_discard,
+        .zerofill    = afr_zerofill,
 
         /* inode read */
         .access      = afr_access,
@@ -500,33 +511,79 @@ struct xlator_cbks cbks = {
 
 struct volume_options options[] = {
         { .key  = {"read-subvolume" },
-          .type = GF_OPTION_TYPE_XLATOR
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. Afr will prefer the one specified using "
+                         "this option if it is not stale. Option value must be "
+                         "one of the xlator names of the children. "
+                         "Ex: <volname>-client-0 till "
+                         "<volname>-client-<number-of-bricks - 1>"
+        },
+        { .key  = {"read-subvolume-index" },
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "-1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one specified using "
+                         "this option if it is not stale. allowed options"
+                         " include -1 till replica-count - 1"
+        },
+        { .key = {"read-hash-mode" },
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 2,
+          .default_value = "1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one computed using "
+                         "the method specified using this option"
+                         "0 = first up server, "
+                         "1 = hash by GFID of file (all clients use "
+                                                    "same subvolume), "
+                         "2 = hash by GFID of file and client PID",
+        },
+        { .key  = {"choose-local" },
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "true",
+          .description = "Choose a local subvolume (i.e. Brick) to read from"
+	                 " if read-subvolume is not explicitly set.",
         },
         { .key  = {"favorite-child"},
-          .type = GF_OPTION_TYPE_XLATOR
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "If a split-brain happens choose subvol/brick set by "
+                         "this option as source."
         },
         { .key  = {"background-self-heal-count"},
           .type = GF_OPTION_TYPE_INT,
           .min  = 0,
           .default_value = "16",
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "This specifies the number of self-heals that can be "
+                         " performed in background without blocking the fop"
         },
         { .key  = {"data-self-heal"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
           .value = {"1", "on", "yes", "true", "enable",
                     "0", "off", "no", "false", "disable",
                     "open"},
           .default_value = "on",
+          .description   = "Using this option we can enable/disable data "
+                           "self-heal on the file. \"open\" means data "
+                           "self-heal action will only be triggered by file "
+                           "open operations."
         },
         { .key  = {"data-self-heal-algorithm"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
           .description   = "Select between \"full\", \"diff\". The "
                            "\"full\" algorithm copies the entire file from "
                            "source to sink. The \"diff\" algorithm copies to "
                            "sink only those blocks whose checksums don't match "
-                           "with those of source.",
-          .value = { "diff", "full", "" }
+                           "with those of source. If no option is configured "
+                           "the option is chosen dynamically as follows: "
+                           "If the file does not exist on one of the sinks "
+                           "or empty file exists or if the source file size is "
+                           "about the same as page size the entire file will "
+                           "be read and written i.e \"full\" algo, "
+                           "otherwise \"diff\" algo is chosen.",
+          .value = { "diff", "full"}
         },
         { .key  = {"data-self-heal-window-size"},
           .type = GF_OPTION_TYPE_INT,
@@ -539,50 +596,102 @@ struct volume_options options[] = {
         { .key  = {"metadata-self-heal"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Using this option we can enable/disable metadata "
+                         "i.e. Permissions, ownerships, xattrs self-heal on "
+                         "the file/directory."
         },
         { .key  = {"entry-self-heal"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Using this option we can enable/disable entry "
+                         "self-heal on the directory."
         },
         { .key  = {"data-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Data fops like write/truncate will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"metadata-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Metadata fops like setattr/setxattr will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"entry-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Entry fops like create/unlink will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"optimistic-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
-        },
-        { .key  = {"strict-readdir"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
+          .description = "Entry/Metadata fops will not perform "
+                         "pre fop changelog operations in afr transaction "
+                         "if this option is enabled."
         },
         { .key = {"inodelk-trace"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "Enabling this option logs inode lock/unlocks"
         },
         { .key = {"entrylk-trace"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "Enabling this option logs entry lock/unlocks"
+        },
+	{ .key = {"pre-op-compat"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "on",
+	  .description = "Use separate pre-op xattrop() FOP rather than "
+	                 "overloading xdata of the OP"
+	},
+        { .key = {"eager-lock"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Lock phase of a transaction has two sub-phases. "
+                         "First is an attempt to acquire locks in parallel by "
+                         "broadcasting non-blocking lock requests. If lock "
+                         "acquisition fails on any server, then the held locks "
+                         "are unlocked and revert to a blocking locked mode "
+                         "sequentially on one server after another.  If this "
+                         "option is enabled the initial broadcasting lock "
+                         "request attempt to acquire lock on the entire file. "
+                         "If this fails, we revert back to the sequential "
+                         "\"regional\" blocking lock as before. In the case "
+                         "where such an \"eager\" lock is granted in the "
+                         "non-blocking phase, it gives rise to an opportunity "
+                         "for optimization. i.e, if the next write transaction "
+                         "on the same FD arrives before the unlock phase of "
+                         "the first transaction, it \"takes over\" the full "
+                         "file lock. Similarly if yet another data transaction "
+                         "arrives before the unlock phase of the \"optimized\" "
+                         "transaction, that in turn \"takes over\" the lock as "
+                         "well. The actual unlock now happens at the end of "
+                         "the last \"optimized\" transaction."
+
         },
         { .key = {"self-heal-daemon"},
           .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
+          .default_value = "on",
+          .description = "This option applies to only self-heal-daemon. "
+                         "Index directory crawl and automatic healing of files "
+                         "will not be performed if this option is turned off."
         },
         { .key = {"iam-self-heal-daemon"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "This option differentiates if the replicate "
+                         "translator is running as part of self-heal-daemon "
+                         "or not."
         },
         { .key = {"quorum-type"},
           .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "auto", "fixed", "" },
+          .value = { "none", "auto", "fixed"},
           .default_value = "none",
           .description = "If value is \"fixed\" only allow writes if "
                          "quorum-count bricks are present.  If value is "
@@ -599,5 +708,42 @@ struct volume_options options[] = {
                          "this many bricks or present.  Other quorum types "
                          "will OVERWRITE this value.",
         },
+        { .key  = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Local glusterd uuid string, used in starting "
+                         "self-heal-daemon so that it can crawl only on "
+                         "local index directories.",
+        },
+        { .key  = {"post-op-delay-secs"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 0,
+          .max  = INT_MAX,
+          .default_value = "1",
+          .description = "Time interval induced artificially before "
+	                 "post-operation phase of the transaction to "
+                         "enhance overlap of adjacent write operations.",
+        },
+        { .key = {AFR_SH_READDIR_SIZE_KEY},
+          .type = GF_OPTION_TYPE_SIZET,
+          .description = "readdirp size for performing entry self-heal",
+          .min = 1024,
+          .max = 131072,
+          .default_value = "1KB",
+        },
+        { .key = {"ensure-durability"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "Afr performs fsyncs for transactions if this "
+                         "option is on to make sure the changelogs/data is "
+                         "written to the disk",
+          .default_value = "on",
+        },
+	{ .key = {"afr-dirty-xattr"},
+	  .type = GF_OPTION_TYPE_STR,
+	  .default_value = AFR_DIRTY_DEFAULT,
+	},
+	{ .key = {"metadata-splitbrain-forced-heal"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "off",
+	},
         { .key  = {NULL} },
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index a1c1bb398..36042f7b2 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -29,94 +20,42 @@
 #include "call-stub.h"
 #include "compat-errno.h"
 #include "afr-mem-types.h"
-#include "afr-self-heal-algorithm.h"
 
 #include "libxlator.h"
 #include "timer.h"
+#include "syncop.h"
+
+#include "afr-self-heald.h"
 
 #define AFR_XATTR_PREFIX "trusted.afr"
 #define AFR_PATHINFO_HEADER "REPLICATE:"
+#define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
+#define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
+#define AFR_DIRTY_DEFAULT AFR_XATTR_PREFIX ".dirty"
+#define AFR_DIRTY (((afr_private_t *) (THIS->private))->afr_dirty)
 
-struct _pump_private;
-
-typedef int (*afr_expunge_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                       int child, int32_t op_error,
-                                       int32_t op_errno);
-
-typedef int (*afr_impunge_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                       int32_t op_error, int32_t op_errno);
-typedef int (*afr_post_remove_call_t) (call_frame_t *frame, xlator_t *this);
+#define AFR_LOCKEE_COUNT_MAX    3
+#define AFR_DOM_COUNT_MAX    3
+#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
 
 typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
-typedef void (*afr_lookup_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                      int32_t op_ret, int32_t op_errno);
-
-typedef enum {
-        AFR_POS_UNKNOWN,
-        AFR_POS_LOCAL,
-        AFR_POS_REMOTE
-} afr_child_pos_t;
 
-typedef enum {
-        SPLIT_BRAIN = 1,
-        ALL_FOOLS = 2
-} afr_subvol_status_t;
+typedef int (*afr_read_txn_wind_t) (call_frame_t *frame, xlator_t *this, int subvol);
 
-typedef enum {
-        AFR_INODE_SET_READ_CTX = 1,
-        AFR_INODE_RM_STALE_CHILDREN,
-        AFR_INODE_SET_OPENDIR_DONE,
-        AFR_INODE_SET_SPLIT_BRAIN,
-        AFR_INODE_GET_READ_CTX,
-        AFR_INODE_GET_OPENDIR_DONE,
-        AFR_INODE_GET_SPLIT_BRAIN,
-} afr_inode_op_t;
-
-typedef struct afr_inode_params_ {
-        afr_inode_op_t op;
-        union {
-                gf_boolean_t value;
-                struct {
-                        int32_t read_child;
-                        int32_t *children;
-                } read_ctx;
-        } u;
-} afr_inode_params_t;
+typedef int (*afr_inode_refresh_cbk_t) (call_frame_t *frame, xlator_t *this, int err);
 
-typedef struct afr_inode_ctx_ {
-        uint64_t masks;
-        int32_t  *fresh_children;//increasing order of latency
-} afr_inode_ctx_t;
+typedef int (*afr_changelog_resume_t) (call_frame_t *frame, xlator_t *this);
 
-typedef enum {
-        NONE,
-        INDEX,
-        FULL,
-} afr_crawl_type_t;
-
-typedef struct afr_self_heald_ {
-        gf_boolean_t     enabled;
-        gf_boolean_t     iamshd;
-        afr_crawl_type_t *pending;
-        gf_boolean_t     *inprogress;
-        afr_child_pos_t  *pos;
-        time_t           *sh_times;
-        gf_timer_t       **timer;
-        eh_t             *healed;
-        eh_t             *heal_failed;
-        eh_t             *split_brain;
-} afr_self_heald_t;
+#define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr;})
+#define AFR_COUNT(array,max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res;})
+#define AFR_INTERSECT(dst,src1,src2,max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i];})
 
 typedef struct _afr_private {
         gf_lock_t lock;               /* to guard access to child_count, etc */
         unsigned int child_count;     /* total number of children   */
 
-        unsigned int read_child_rr;   /* round-robin index of the read_child */
-        gf_lock_t read_child_lock;    /* lock to protect above */
-
         xlator_t **children;
 
-        int first_lookup;
         inode_t *root_inode;
 
         unsigned char *child_up;
@@ -137,140 +76,54 @@ typedef struct _afr_private {
         gf_boolean_t metadata_change_log;   /* on/off */
         gf_boolean_t entry_change_log;      /* on/off */
 
+	gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
         int read_child;               /* read-subvolume */
+        unsigned int hash_mode;       /* for when read_child is not set */
         int favorite_child;  /* subvolume to be preferred in resolving
                                          split-brain cases */
 
         gf_boolean_t inodelk_trace;
         gf_boolean_t entrylk_trace;
 
-        gf_boolean_t strict_readdir;
-
         unsigned int wait_count;      /* # of servers to wait for success */
 
         uint64_t up_count;      /* number of CHILD_UPs we have seen */
         uint64_t down_count;    /* number of CHILD_DOWNs we have seen */
 
-        struct _pump_private *pump_private; /* Set if we are loaded as pump */
-        int                   use_afr_in_pump;
-
-        pthread_mutex_t  mutex;
-        struct list_head saved_fds;   /* list of fds on which locks have succeeded */
         gf_boolean_t      optimistic_change_log;
         gf_boolean_t      eager_lock;
+        gf_boolean_t      pre_op_compat;      /* on/off */
+	uint32_t          post_op_delay_secs;
         unsigned int      quorum_count;
 
         char                   vol_uuid[UUID_SIZE + 1];
         int32_t                *last_event;
-        afr_self_heald_t       shd;
-} afr_private_t;
-
-typedef struct {
-        /* External interface: These are variables (some optional) that
-           are set by whoever has triggered self-heal */
-
-        gf_boolean_t do_data_self_heal;
-        gf_boolean_t do_metadata_self_heal;
-        gf_boolean_t do_entry_self_heal;
-        gf_boolean_t do_gfid_self_heal;
-        gf_boolean_t do_missing_entry_self_heal;
-
-        gf_boolean_t forced_merge;        /* Is this a self-heal triggered to
-                                             forcibly merge the directories? */
-
-        gf_boolean_t background;          /* do self-heal in background
-                                             if possible */
-        ia_type_t type;                   /* st_mode of the entry we're doing
-                                             self-heal on */
-        inode_t   *inode;                 /* inode on which the self-heal is
-                                             performed on */
-        uuid_t  sh_gfid_req;                 /* gfid self-heal needs to be done
-                                             with this gfid if it is not null */
-
-        /* Function to call to unwind. If self-heal is being done in the
-           background, this function will be called as soon as possible. */
-
-        int (*unwind) (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno);
-
-        /* End of external interface members */
-
-
-        /* array of stat's, one for each child */
-        struct iatt *buf;
-        struct iatt *parentbufs;
-        struct iatt parentbuf;
-        struct iatt entrybuf;
-
-        afr_expunge_done_cbk_t expunge_done;
-        afr_impunge_done_cbk_t impunge_done;
-
-        /* array of xattr's, one for each child */
-        dict_t **xattr;
-
-        /* array containing if the lookups succeeded in the order of response
-         */
-        int32_t *success_children;
-        int     success_count;
-        /* array containing the fresh children found in the self-heal process */
-        int32_t *fresh_children;
-        /* array containing the fresh children found in the parent lookup */
-        int32_t *fresh_parent_dirs;
-        /* array of errno's, one for each child */
-        int *child_errno;
-        /*loc used for lookup*/
-        loc_t lookup_loc;
-        int32_t lookup_flags;
-        afr_lookup_done_cbk_t lookup_done;
-
-        int32_t **pending_matrix;
-        int32_t **delta_matrix;
 
-        int32_t op_ret;
-        int32_t op_errno;
-
-        int *sources;
-        int source;
-        int active_source;
-        int active_sinks;
-        unsigned char *success;
-        unsigned char *locked_nodes;
-        int lock_count;
-
-        const char *linkname;
-        gf_boolean_t entries_skipped;
-
-        int   op_failed;
-
-        gf_boolean_t data_lock_held;
-        gf_boolean_t eof_reached;
-        fd_t  *healing_fd;
-        int   file_has_holes;
-        blksize_t block_size;
-        off_t file_size;
-        off_t offset;
-        unsigned char *write_needed;
-        uint8_t *checksum;
-        afr_post_remove_call_t post_remove_call;
-
-        loc_t parent_loc;
-
-        call_frame_t *orig_frame;
-        call_frame_t *old_loop_frame;
-        gf_boolean_t unwound;
-
-        afr_sh_algo_private_t *private;
-
-        afr_lock_cbk_t data_lock_success_handler;
-        afr_lock_cbk_t data_lock_failure_handler;
-        int (*completion_cbk) (call_frame_t *frame, xlator_t *this);
-        int (*sh_data_algo_start) (call_frame_t *frame, xlator_t *this);
-        int (*algo_completion_cbk) (call_frame_t *frame, xlator_t *this);
-        int (*algo_abort_cbk) (call_frame_t *frame, xlator_t *this);
-        void (*gfid_sh_success_cbk) (call_frame_t *sh_frame, xlator_t *this);
+	/* @event_generation: Keeps count of number of events received which can
+	   potentially impact consistency decisions. The events are CHILD_UP
+	   and CHILD_DOWN, when we have to recalculate the freshness/staleness
+	   of copies to detect if changes had happened while the other server
+	   was down. CHILD_DOWN and CHILD_UP can also be received on network
+	   disconnect/reconnects and not necessarily server going down/up.
+	   Recalculating freshness/staleness on network events is equally
+	   important as we might have had a network split brain.
+	*/
+	uint32_t               event_generation;
+
+        gf_boolean_t           choose_local;
+        gf_boolean_t           did_discovery;
+        uint64_t               sh_readdir_size;
+        gf_boolean_t           ensure_durability;
+        char                   *sh_domain;
+	char                   *afr_dirty;
+
+	afr_self_heald_t       shd;
+
+	/* pump dependencies */
+	void                   *pump_private;
+	gf_boolean_t           use_afr_in_pump;
+} afr_private_t;
 
-        call_frame_t *sh_frame;
-} afr_self_heal_t;
 
 typedef enum {
         AFR_DATA_TRANSACTION,          /* truncate, write, ... */
@@ -332,11 +185,31 @@ afr_index_for_transaction_type (afr_transaction_type type)
         return -1;  /* make gcc happy */
 }
 
+typedef struct {
+        loc_t                   loc;
+        char                    *basename;
+        unsigned char           *locked_nodes;
+        int                     locked_count;
+
+} afr_entry_lockee_t;
+
+int
+afr_entry_lockee_cmp (const void *l1, const void *l2);
+
+typedef struct {
+        char    *domain; /* Domain on which inodelk is taken */
+        struct gf_flock flock;
+        unsigned char *locked_nodes;
+        int32_t lock_count;
+} afr_inodelk_t;
 
 typedef struct {
         loc_t *lk_loc;
-        struct gf_flock lk_flock;
 
+        int                     lockee_count;
+        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
+
+        afr_inodelk_t       inodelk[AFR_DOM_COUNT_MAX];
         const char *lk_basename;
         const char *lower_basename;
         const char *higher_basename;
@@ -345,87 +218,203 @@ typedef struct {
 
         unsigned char *locked_nodes;
         unsigned char *lower_locked_nodes;
-        unsigned char *inode_locked_nodes;
-        unsigned char *entry_locked_nodes;
 
         selfheal_lk_type_t selfheal_lk_type;
         transaction_lk_type_t transaction_lk_type;
 
         int32_t lock_count;
-        int32_t inodelk_lock_count;
         int32_t entrylk_lock_count;
 
         uint64_t lock_number;
         int32_t lk_call_count;
         int32_t lk_expected_count;
+        int32_t lk_attempted_count;
 
         int32_t lock_op_ret;
         int32_t lock_op_errno;
         afr_lock_cbk_t lock_cbk;
+        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
 } afr_internal_lock_t;
 
-typedef struct _afr_locked_fd {
-        fd_t  *fd;
-        struct list_head list;
-} afr_locked_fd_t;
+struct afr_reply {
+	int	valid;
+	int32_t	op_ret;
+	int32_t	op_errno;
+	dict_t *xdata;
+	struct iatt poststat;
+	struct iatt postparent;
+	struct iatt prestat;
+	struct iatt preparent;
+	struct iatt preparent2;
+	struct iatt postparent2;
+	uint8_t checksum[MD5_DIGEST_LENGTH];
+};
+
+typedef enum {
+        AFR_FD_NOT_OPENED,
+        AFR_FD_OPENED,
+        AFR_FD_OPENING
+} afr_fd_open_status_t;
+
+typedef struct {
+        unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
+	int inherited[AFR_NUM_CHANGE_LOGS];
+	int on_disk[AFR_NUM_CHANGE_LOGS];
+        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
+
+        unsigned int *lock_piggyback;
+        unsigned int *lock_acquired;
+
+        int flags;
+
+	/* used for delayed-post-op optimization */
+	pthread_mutex_t    delay_lock;
+	gf_timer_t        *delay_timer;
+	call_frame_t      *delay_frame;
+
+	/* set if any write on this fd was a non stable write
+	   (i.e, without O_SYNC or O_DSYNC)
+	*/
+	gf_boolean_t      witnessed_unstable_write;
+
+	/* @open_fd_count:
+	   Number of open FDs queried from the server, as queried through
+	   xdata in FOPs. Currently, used to decide if eager-locking must be
+	   temporarily disabled.
+	*/
+        uint32_t        open_fd_count;
+
+
+	/* list of frames currently in progress */
+	struct list_head  eager_locked;
+} afr_fd_ctx_t;
+
 
 typedef struct _afr_local {
-        int     uid;
-        int     gid;
+	glusterfs_fop_t  op;
         unsigned int call_count;
-        unsigned int success_count;
-        unsigned int enoent_count;
-
 
-        unsigned int govinda_gOvinda;
+	/* @event_generation: copy of priv->event_generation taken at the
+	   time of starting the transaction. The copy is made so that we
+	   have a stable value through the various phases of the transaction.
+	*/
+	unsigned int event_generation;
 
-        unsigned int read_child_index;
-        unsigned char read_child_returned;
-        unsigned int first_up_child;
+        uint32_t     open_fd_count;
+        gf_boolean_t update_open_fd_count;
 
-        pid_t saved_pid;
+	gf_lkowner_t  saved_lk_owner;
 
         int32_t op_ret;
         int32_t op_errno;
 
         int32_t **pending;
 
+	int dirty[AFR_NUM_CHANGE_LOGS];
+
         loc_t loc;
         loc_t newloc;
 
         fd_t *fd;
-        unsigned char *fd_open_on;
-
-        glusterfs_fop_t fop;
+	afr_fd_ctx_t *fd_ctx;
 
+	/* @child_up: copy of priv->child_up taken at the time of transaction
+	   start. The copy is taken so that we have a stable child_up array
+	   through the phases of the transaction as priv->child_up[i] can keep
+	   changing through time.
+	*/
         unsigned char *child_up;
-        int32_t       *fresh_children; //in the order of response
 
-        int32_t *child_errno;
+	/* @read_attempted:
+	   array of flags representing subvolumes where read operations of
+	   the read transaction have already been attempted. The array is
+	   first pre-filled with down subvolumes, and as reads are performed
+	   on other subvolumes, those are set as well. This way if the read
+	   operation fails we do not retry on that subvolume again.
+	*/
+	unsigned char *read_attempted;
 
-        dict_t  *xattr_req;
+	/* @readfn:
 
-        int32_t  inodelk_count;
-        int32_t  entrylk_count;
+	   pointer to function which will perform the read operation on a given
+	   subvolume. Used in read transactions.
+	*/
 
-        afr_internal_lock_t internal_lock;
+	afr_read_txn_wind_t readfn;
+
+	/* @refreshed:
+
+	   the inode was "refreshed" (i.e, pending xattrs from all subvols
+	   freshly inspected and inode ctx updated accordingly) as part of
+	   this transaction already.
+	*/
+	gf_boolean_t refreshed;
+
+	/* @inode:
+
+	   the inode on which the read txn is performed on. ref'ed and copied
+	   from either fd->inode or loc.inode
+	*/
+
+	inode_t *inode;
+
+	/* @parent[2]:
 
-        afr_locked_fd_t *locked_fd;
-        int32_t          source_child;
-        int32_t          lock_recovery_child;
+	   parent inode[s] on which directory transactions are performed.
+	*/
+
+	inode_t *parent;
+	inode_t *parent2;
+
+	/* @readable:
+
+	   array of flags representing servers from which a read can be
+	   performed. This is the output of afr_inode_refresh()
+	*/
+	unsigned char *readable;
+
+	afr_inode_refresh_cbk_t refreshfn;
+
+	/* @refreshinode:
+
+	   Inode currently getting refreshed.
+	*/
+	inode_t *refreshinode;
+
+	/*
+	  @pre_op_compat:
+
+	  compatibility mode of pre-op. send a separate pre-op and
+	  op operations as part of transaction, rather than combining
+	*/
+
+	gf_boolean_t pre_op_compat;
+
+        dict_t  *xattr_req;
+
+        afr_internal_lock_t internal_lock;
 
         dict_t  *dict;
+
         int      optimistic_change_log;
+	gf_boolean_t      delayed_post_op;
 
-        gf_boolean_t    fop_paused;
-        int (*fop_call_continue) (call_frame_t *frame, xlator_t *this);
+	/* Is the current writev() going to perform a stable write?
+	   i.e, is fd->flags or @flags writev param have O_SYNC or
+	   O_DSYNC?
+	*/
+	gf_boolean_t      stable_write;
+
+	/* This write appended to the file. Nnot necessarily O_APPEND,
+	   just means the offset of write was at the end of file.
+	*/
+	gf_boolean_t      append_write;
 
         /*
           This struct contains the arguments for the "continuation"
           (scheme-like) of fops
         */
 
-        int   op;
         struct {
                 struct {
                         unsigned char buf_set;
@@ -433,22 +422,7 @@ typedef struct _afr_local {
                 } statfs;
 
                 struct {
-                        uuid_t  gfid_req;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt postparent;
-                        dict_t **xattrs;
-                        dict_t *xattr;
-                        struct iatt *postparents;
-                        struct iatt *bufs;
-                        int32_t read_child;
-                        int32_t *sources;
-                        int32_t *success_children;
-                } lookup;
-
-                struct {
                         int32_t flags;
-                        int32_t wbflags;
                 } open;
 
                 struct {
@@ -515,7 +489,9 @@ typedef struct _afr_local {
                 struct {
                         struct iatt prebuf;
                         struct iatt postbuf;
+                } inode_wfop; //common structure for all inode-write-fops
 
+                struct {
                         int32_t op_ret;
 
                         struct iovec *vector;
@@ -526,34 +502,21 @@ typedef struct _afr_local {
                 } writev;
 
                 struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } fsync;
-
-                struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } truncate;
 
                 struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } ftruncate;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } setattr;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } fsetattr;
 
                 struct {
@@ -570,93 +533,77 @@ typedef struct _afr_local {
                         char *name;
                 } removexattr;
 
+                struct {
+                        dict_t *xattr;
+                } xattrop;
+
+                struct {
+                        dict_t *xattr;
+                } fxattrop;
+
                 /* dir write */
 
                 struct {
-                        fd_t *fd;
-                        dict_t *params;
-                        int32_t flags;
-                        mode_t mode;
                         inode_t *inode;
                         struct iatt buf;
                         struct iatt preparent;
                         struct iatt postparent;
-                        struct iatt read_child_buf;
+                        struct iatt prenewparent;
+                        struct iatt postnewparent;
+                } dir_fop; //common structure for all dir fops
+
+                struct {
+                        fd_t *fd;
+                        dict_t *params;
+                        int32_t flags;
+                        mode_t mode;
                 } create;
 
                 struct {
                         dev_t dev;
                         mode_t mode;
                         dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt read_child_buf;
                 } mknod;
 
                 struct {
                         int32_t mode;
                         dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
                 } mkdir;
 
                 struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } unlink;
-
-                struct {
-                        int   flags;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
+                        int flags;
                 } rmdir;
 
                 struct {
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preoldparent;
-                        struct iatt prenewparent;
-                        struct iatt postoldparent;
-                        struct iatt postnewparent;
-                } rename;
-
-                struct {
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } link;
-
-                struct {
-                        inode_t *inode;
                         dict_t *params;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
                         char *linkpath;
-                        struct iatt preparent;
-                        struct iatt postparent;
                 } symlink;
 
+		struct {
+			int32_t mode;
+			off_t offset;
+			size_t len;
+		} fallocate;
+
+		struct {
+			off_t offset;
+			size_t len;
+		} discard;
+
                 struct {
-                        int32_t flags;
-                        dir_entry_t *entries;
-                        int32_t count;
-                } setdents;
+                        off_t offset;
+                        off_t len;
+                        struct iatt prebuf;
+                        struct iatt postbuf;
+                } zerofill;
+
+
         } cont;
 
         struct {
                 off_t start, len;
 
+                gf_boolean_t    eager_lock_on;
                 int *eager_lock;
 
                 char *basename;
@@ -667,16 +614,67 @@ typedef struct _afr_local {
 
                 afr_transaction_type type;
 
-                int success_count;
-                int erase_pending;
-                int failure_count;
+		/* stub to resume on destruction
+		   of the transaction frame */
+		call_stub_t      *resume_stub;
+
+		struct list_head  eager_locked;
 
-                int last_tried;
-                int32_t *child_errno;
                 unsigned char   *pre_op;
 
+		/* @fop_subvols: subvolumes on which FOP will be attempted */
+                unsigned char   *fop_subvols;
+
+		/* @failed_subvols: subvolumes on which FOP failed. Always
+		   a subset of @fop_subvols */
+                unsigned char   *failed_subvols;
+
+		/* @dirtied: flag which indicates whether we set dirty flag
+		   in the OP. Typically true when we are performing operation
+		   on more than one subvol and optimistic changelog is disabled
+
+		   A 'true' value set in @dirtied flag means an 'undirtying'
+		   has to be done in POST-OP phase.
+		*/
+		gf_boolean_t  dirtied;
+
+		/* @inherited: flag which indicates that the dirty flags
+		   of the previous transaction were inherited
+		*/
+		gf_boolean_t  inherited;
+
+		/*
+		  @no_uninherit: flag which indicates that a pre_op_uninherit()
+		  must _not_ be attempted (and returned as failure) always. This
+		  flag is set when a hard pre-op is performed, but not accounted
+		  for it in fd_ctx->on_disk[]. Such transactions are "isolated"
+		  from the pre-op piggybacking entirely and therefore uninherit
+		  must not be attempted.
+		*/
+		gf_boolean_t no_uninherit;
+
+		/* @uninherit_done:
+		   @uninherit_value:
+
+		   The above pair variables make pre_op_uninherit() idempotent.
+		   Both are FALSE initially. The first call to pre_op_uninherit
+		   sets @uninherit_done to TRUE and the return value to
+		   @uninherit_value. Further calls will check for @uninherit_done
+		   to be TRUE and if so will simply return @uninherit_value.
+		*/
+		gf_boolean_t uninherit_done;
+		gf_boolean_t uninherit_value;
+
+		/* @changelog_resume: function to be called after changlogging
+		   (either pre-op or post-op) is done
+		*/
+
+		afr_changelog_resume_t changelog_resume;
+
                 call_frame_t *main_frame;
 
+                int (*wind) (call_frame_t *frame, xlator_t *this, int subvol);
+
                 int (*fop) (call_frame_t *frame, xlator_t *this);
 
                 int (*done) (call_frame_t *frame, xlator_t *this);
@@ -688,56 +686,19 @@ typedef struct _afr_local {
                 /* post-op hook */
         } transaction;
 
-        afr_self_heal_t self_heal;
+	syncbarrier_t barrier;
 
         struct marker_str     marker;
-} afr_local_t;
-
-typedef enum {
-        AFR_FD_NOT_OPENED,
-        AFR_FD_OPENED,
-        AFR_FD_OPENING
-} afr_fd_open_status_t;
-
-typedef struct {
-        struct list_head call_list;
-        call_frame_t    *frame;
-} afr_fd_paused_call_t;
-
-typedef struct {
-        unsigned int *pre_op_done;
-        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
-        unsigned int *pre_op_piggyback;
-
-        unsigned int *lock_piggyback;
-        unsigned int *lock_acquired;
-
-        int flags;
-        int32_t wbflags;
-        uint64_t up_count;   /* number of CHILD_UPs this fd has seen */
-        uint64_t down_count; /* number of CHILD_DOWNs this fd has seen */
-
-        int32_t last_tried;
-
-        int  hit, miss;
-        gf_boolean_t failed_over;
-        struct list_head entries; /* needed for readdir failover */
-
-        unsigned char *locked_on; /* which subvolumes locks have been successful */
-	struct list_head  paused_calls; /* queued calls while fix_open happens  */
-} afr_fd_ctx_t;
 
+        /* extra data for fops */
+        dict_t         *xdata_req;
+        dict_t         *xdata_rsp;
 
-/* try alloc and if it fails, goto label */
-#define AFR_LOCAL_ALLOC_OR_GOTO(var, label) do {                    \
-                var = mem_get0 (THIS->local_pool);                  \
-                if (!var) {                                         \
-                        gf_log (this->name, GF_LOG_ERROR,           \
-                                "out of memory :(");                \
-                        op_errno = ENOMEM;                          \
-                        goto label;                                 \
-                }                                                   \
-        } while (0);
+        mode_t          umask;
+        int             xflag;
+        gf_boolean_t    do_discovery;
+	struct afr_reply *replies;
+} afr_local_t;
 
 
 /* did a call fail due to a child failing? */
@@ -745,40 +706,76 @@ typedef struct {
                                            ((op_errno == ENOTCONN) ||   \
                                             (op_errno == EBADFD)))
 
-#define afr_fop_failed(op_ret, op_errno) ((op_ret) == -1)
+int
+afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			   unsigned char *data_subvols,
+			   unsigned char *metadata_subvols,
+			   int *event_generation);
+int
+__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			     unsigned char *data_subvols,
+			     unsigned char *metadata_subvols,
+			     int *event_generation);
 
-/* have we tried all children? */
-#define all_tried(i, count)  ((i) == (count) - 1)
+int
+__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
+			     unsigned char *data_subvols,
+			     unsigned char *metadata_subvol,
+			     int event_generation);
+int
+afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
+			   unsigned char *data_subvols,
+			   unsigned char *metadata_subvols,
+			   int event_generation);
 
-int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid);
+int
+afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this);
 
 int
-pump_command_reply (call_frame_t *frame, xlator_t *this);
+afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
+				  unsigned char *readable);
+
+int
+afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
+				unsigned char *readable, int *event_p,
+				int type);
+int
+afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
+		     int *event_p, afr_transaction_type type);
+
+#define afr_data_subvol_get(i, t, s, e) \
+	afr_read_subvol_get(i, t, s, e, AFR_DATA_TRANSACTION)
+
+#define afr_metadata_subvol_get(i, t, s, e) \
+	afr_read_subvol_get(i, t, s, e, AFR_METADATA_TRANSACTION)
+
+int
+afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		   afr_inode_refresh_cbk_t cbk);
 
 int32_t
 afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
 
 int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count);
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
 
 int
-afr_save_locked_fd (xlator_t *this, fd_t *fd);
+afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
 
 int
 afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
                        unsigned char *locked_nodes);
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this);
+afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
 
 int
 afr_set_lock_number (call_frame_t *frame, xlator_t *this);
 
-
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2);
-
 int32_t
 afr_unlock (call_frame_t *frame, xlator_t *this);
 
@@ -794,46 +791,30 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this);
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
 
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
                           unsigned int child_count);
 
-int pump_start (call_frame_t *frame, xlator_t *this);
-
 int
 __afr_fd_ctx_set (xlator_t *this, fd_t *fd);
 
 int
 afr_fd_ctx_set (xlator_t *this, fd_t *fd);
 
-int32_t
-afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children);
-
-void
-afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
-                        int32_t *fresh_children);
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 
 int
 afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
 
-unsigned int
-afr_up_children_count (unsigned char *child_up, unsigned int child_count);
-
-unsigned int
-afr_locked_children_count (unsigned char *children, unsigned int child_count);
-
-unsigned int
-afr_pre_op_done_children_count (unsigned char *pre_op,
-                                unsigned int child_count);
+int
+afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
 
-gf_boolean_t
-afr_is_fresh_lookup (loc_t *loc, xlator_t *this);
+int
+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 void
-afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent);
-
-int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
+afr_replies_wipe (afr_local_t *local, afr_private_t *priv);
 
 void
 afr_local_cleanup (afr_local_t *local, xlator_t *this);
@@ -841,21 +822,9 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this);
 int
 afr_frame_return (call_frame_t *frame);
 
-gf_boolean_t
-afr_is_split_brain (xlator_t *this, inode_t *inode);
-
-void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set);
-
 int
 afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags);
-
-void
-afr_set_opendir_done (xlator_t *this, inode_t *inode);
-
-gf_boolean_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode);
+          fd_t *fd, dict_t *xdata);
 
 void
 afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
@@ -863,9 +832,6 @@ afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
 int
 afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd);
 
-int
-afr_launch_openfd_self_heal (call_frame_t *frame, xlator_t *this, fd_t *fd);
-
 #define AFR_STACK_UNWIND(fop, frame, params ...)                \
         do {                                                    \
                 afr_local_t *__local = NULL;                    \
@@ -896,7 +862,16 @@ afr_launch_openfd_self_heal (call_frame_t *frame, xlator_t *this, fd_t *fd);
                 }                                               \
         } while (0);
 
-#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
+#define AFR_FRAME_INIT(frame, op_errno)				       \
+	({frame->local = mem_get0 (THIS->local_pool);		       \
+	if (afr_local_init (frame->local, THIS->private, &op_errno)) { \
+		afr_local_cleanup (frame->local, THIS);		       \
+		mem_put (frame->local);				       \
+		frame->local = NULL; };				       \
+	frame->local;})
+
+#define AFR_STACK_RESET(frame) do { int opr; STACK_RESET (frame->root); AFR_FRAME_INIT(frame, opr);} while (0)
+
 /* allocate and return a string that is the basename of argument */
 static inline char *
 AFR_BASENAME (const char *str)
@@ -909,6 +884,9 @@ AFR_BASENAME (const char *str)
         return __basename_str;
 }
 
+call_frame_t *
+afr_copy_frame (call_frame_t *base);
+
 int
 afr_transaction_local_init (afr_local_t *local, xlator_t *this);
 
@@ -916,9 +894,6 @@ int32_t
 afr_marker_getxattr (call_frame_t *frame, xlator_t *this,
                      loc_t *loc, const char *name,afr_local_t *local, afr_private_t *priv );
 
-int32_t *
-afr_children_create (int32_t child_count);
-
 int
 afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
 
@@ -927,103 +902,25 @@ afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
                         transaction_lk_type_t lk_type);
 
 int
-afr_first_up_child (unsigned char *child_up, size_t child_count);
+afr_higher_errno (int32_t old_errno, int32_t new_errno);
 
 int
-afr_select_read_child_from_policy (int32_t *fresh_children, int32_t child_count,
-                                   int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources);
+afr_final_errno (afr_local_t *local, afr_private_t *priv);
 
-void
-afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
-                              int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child);
-
-int32_t
-afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
-                    int32_t *fresh_children,
-                    int32_t *call_child, int32_t *last_index);
-
-int32_t
-afr_next_call_child (int32_t *fresh_children, unsigned char *child_up,
-                     size_t child_count, int32_t *last_index,
-                     int32_t read_child);
-void
-afr_get_fresh_children (int32_t *success_children, int32_t *sources,
-                        int32_t *children, unsigned int child_count);
-void
-afr_children_add_child (int32_t *children, int32_t child,
-                              int32_t child_count);
-void
-afr_children_rm_child (int32_t *children, int32_t child,
-                             int32_t child_count);
-void
-afr_reset_children (int32_t *children, int32_t child_count);
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno);
-int
-afr_errno_count (int32_t *children, int *child_errno,
-                 unsigned int child_count, int32_t op_errno);
 int
-afr_get_children_count (int32_t *children, unsigned int child_count);
-gf_boolean_t
-afr_is_child_present (int32_t *success_children, int32_t child_count,
-                      int32_t child);
-void
-afr_update_gfid_from_iatts (uuid_t uuid, struct iatt *bufs,
-                            int32_t *success_children,
-                            unsigned int child_count);
-void
-afr_reset_xattr (dict_t **xattr, unsigned int child_count);
-gf_boolean_t
-afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
-                        unsigned int child_count, const char *path,
-                        const char *xlator_name);
-unsigned int
-afr_gfid_missing_count (const char *xlator_name, int32_t *children,
-                        struct iatt *bufs, unsigned int child_count,
-                        const char *path);
-void
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path);
-void
-afr_children_copy (int32_t *dst, int32_t *src, unsigned int child_count);
-afr_transaction_type
-afr_transaction_type_get (ia_type_t ia_type);
-int32_t
-afr_resultant_errno_get (int32_t *children,
-                         int *child_errno, unsigned int child_count);
-void
-afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
-                             int32_t *stale_children);
+afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req);
+
 void
-afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                      gf_boolean_t background, ia_type_t ia_type, char *reason,
-                      void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
-                                                   xlator_t *this),
-                      int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno));
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open);
-int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop);
-int
-afr_set_elem_count_get (unsigned char *elems, int child_count);
+afr_fix_open (fd_t *fd, xlator_t *this);
 
 afr_fd_ctx_t *
 afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 
-gf_boolean_t
-afr_open_only_data_self_heal (char *data_self_heal);
-
-gf_boolean_t
-afr_data_self_heal_enabled (char *data_self_heal);
-
 void
 afr_set_low_priority (call_frame_t *frame);
 int
 afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags, int32_t wb_flags);
+                      int flags);
 
 gf_boolean_t
 afr_have_quorum (char *logname, afr_private_t *priv);
@@ -1033,6 +930,10 @@ afr_matrix_cleanup (int32_t **pending, unsigned int m);
 
 int32_t**
 afr_matrix_create (unsigned int m, unsigned int n);
+
+void
+afr_filter_xattrs (dict_t *xattr);
+
 /*
  * Special value indicating we should use the "auto" quorum method instead of
  * a fixed value (including zero to turn off quorum enforcement).
@@ -1052,4 +953,24 @@ afr_matrix_create (unsigned int m, unsigned int n);
         }                                                                \
 } while (0);
 
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
+
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd);
+
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub);
+
+int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count);
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this);
+
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *is_local);
+
+void
+afr_remove_eager_lock_stub (afr_local_t *local);
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index 9c6f9a52a..eed509956 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <unistd.h>
@@ -30,6 +21,120 @@
 #include "afr-common.c"
 #include "defaults.c"
 #include "glusterfs.h"
+#include "pump.h"
+
+
+static int
+afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
+{
+        int ret       = 0;
+        uuid_t *pgfid = NULL;
+
+        GF_ASSERT (gfid);
+
+        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
+        if (!pgfid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*pgfid, gfid);
+
+        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "gfid set failed");
+
+out:
+        if (ret && pgfid)
+                GF_FREE (pgfid);
+	return ret;
+}
+
+static int
+afr_set_root_gfid (dict_t *dict)
+{
+        uuid_t gfid;
+        int ret = 0;
+
+        memset (gfid, 0, 16);
+        gfid[15] = 1;
+
+        ret = afr_set_dict_gfid (dict, gfid);
+
+        return ret;
+}
+
+static int
+afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+{
+        int   ret = -1;
+        uuid_t pargfid = {0};
+
+        if (!child)
+                goto out;
+
+        if (!uuid_is_null (parent->inode->gfid))
+                uuid_copy (pargfid, parent->inode->gfid);
+        else if (!uuid_is_null (parent->gfid))
+                uuid_copy (pargfid, parent->gfid);
+
+        if (uuid_is_null (pargfid))
+                goto out;
+
+        if (strcmp (parent->path, "/") == 0)
+                ret = gf_asprintf ((char **)&child->path, "/%s", name);
+        else
+                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
+                                   name);
+
+        if (-1 == ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "asprintf failed while setting child path");
+        }
+
+        child->name = strrchr (child->path, '/');
+        if (child->name)
+                child->name++;
+
+        child->parent = inode_ref (parent->inode);
+        child->inode = inode_new (parent->inode->table);
+        uuid_copy (child->pargfid, pargfid);
+
+        if (!child->inode) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if ((ret == -1) && child)
+                loc_wipe (child);
+
+        return ret;
+}
+
+static void
+afr_build_root_loc (xlator_t *this, loc_t *loc)
+{
+        afr_private_t   *priv = NULL;
+
+        priv = this->private;
+        loc->path = gf_strdup ("/");
+        loc->name = "";
+        loc->inode = inode_ref (priv->root_inode);
+        uuid_copy (loc->gfid, loc->inode->gfid);
+}
+
+static void
+afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
+{
+        GF_ASSERT (loc);
+        GF_ASSERT (buf);
+
+        uuid_copy (loc->gfid, buf->ia_gfid);
+        if (postparent)
+                uuid_copy (loc->pargfid, postparent->ia_gfid);
+}
 
 static uint64_t pump_pid = 0;
 static inline void
@@ -396,59 +501,77 @@ gf_pump_traverse_directory (loc_t *loc)
                         if (ret)
                                 goto out;
 
-                        if (!IS_ENTRY_CWD (entry->d_name) &&
-                            !IS_ENTRY_PARENT (entry->d_name)) {
-
-                                    is_directory_empty = _gf_false;
-                                    gf_log (this->name, GF_LOG_DEBUG,
-                                            "lookup %s => %"PRId64,
-                                            entry_loc.path,
-                                            iatt.ia_ino);
-
-                                    ret = syncop_lookup (this, &entry_loc, NULL,
-                                                         &iatt, &xattr_rsp, &parent);
-
-                                    if (ret) {
-                                            gf_log (this->name, GF_LOG_ERROR,
-                                                    "%s: lookup failed",
-                                                    entry_loc.path);
-                                            continue;
-                                    }
-                                    pump_fill_loc_info (&entry_loc, &iatt,
-                                                       &parent);
-
-                                    pump_update_resume_state (this, entry_loc.path);
-
-                                    pump_save_path (this, entry_loc.path);
-                                    pump_save_file_stats (this, entry_loc.path);
-
-                                    ret = pump_check_and_update_status (this);
-                                    if (ret < 0) {
-                                            gf_log (this->name, GF_LOG_DEBUG,
-                                                    "Pump beginning to exit out");
-                                            goto out;
-                                    }
-
-                                    if (IA_ISDIR (iatt.ia_type)) {
-                                            if (is_pump_traversal_allowed (this, entry_loc.path)) {
-                                                    gf_log (this->name, GF_LOG_TRACE,
-                                                            "entering dir=%s",
-                                                            entry->d_name);
-                                                    gf_pump_traverse_directory (&entry_loc);
-                                            }
-                                    }
-                            }
+			if ((strcmp (entry->d_name, ".") == 0) ||
+			    (strcmp (entry->d_name, "..") == 0))
+				continue;
+
+			is_directory_empty = _gf_false;
+			gf_log (this->name, GF_LOG_DEBUG,
+				"lookup %s => %"PRId64,
+				entry_loc.path,
+				iatt.ia_ino);
+
+			ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+					     &xattr_rsp, &parent);
+
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: lookup failed", entry_loc.path);
+				continue;
+			}
+
+			ret = afr_selfheal_name (this, loc->gfid, entry->d_name);
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: name self-heal failed (%s/%s)",
+					entry_loc.path, uuid_utoa (loc->gfid),
+					entry->d_name);
+				continue;
+			}
+
+			ret = afr_selfheal (this, iatt.ia_gfid);
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: self-heal failed (%s)",
+					entry_loc.path, uuid_utoa (iatt.ia_gfid));
+				continue;
+			}
+
+			pump_fill_loc_info (&entry_loc, &iatt, &parent);
+
+			pump_update_resume_state (this, entry_loc.path);
+
+			pump_save_path (this, entry_loc.path);
+			pump_save_file_stats (this, entry_loc.path);
+
+			ret = pump_check_and_update_status (this);
+			if (ret < 0) {
+				gf_log (this->name, GF_LOG_DEBUG,
+					"Pump beginning to exit out");
+				goto out;
+			}
+
+			if (IA_ISDIR (iatt.ia_type)) {
+				if (is_pump_traversal_allowed (this, entry_loc.path)) {
+					gf_log (this->name, GF_LOG_TRACE,
+						"entering dir=%s", entry->d_name);
+					gf_pump_traverse_directory (&entry_loc);
+				}
+                        }
                 }
 
                 gf_dirent_free (&entries);
                 free_entries = _gf_false;
-                gf_log (this->name, GF_LOG_TRACE,
-                        "offset incremented to %d",
+                gf_log (this->name, GF_LOG_TRACE, "offset incremented to %d",
                         (int32_t ) offset);
 
         }
 
-        if (is_directory_empty && IS_ROOT_PATH (loc->path)) {
+        ret = syncop_close (fd);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_DEBUG, "closing the fd failed");
+
+        if (is_directory_empty && (strcmp (loc->path, "/") == 0)) {
                pump_change_state (this, PUMP_STATE_RUNNING);
                gf_log (this->name, GF_LOG_INFO, "Empty source brick. "
                                 "Nothing to be done.");
@@ -487,7 +610,7 @@ pump_update_resume_path (xlator_t *this)
 
 static int32_t
 pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_private_t  *priv      = NULL;
         loc_t           loc       = {0};
@@ -501,17 +624,18 @@ pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
         afr_build_root_loc (this, &loc);
 
         ret = syncop_removexattr (priv->children[source], &loc,
-                                          PUMP_PATH);
+				  PUMP_PATH, 0);
 
         ret = syncop_removexattr (priv->children[sink], &loc,
-                                  PUMP_SINK_COMPLETE);
+                                  PUMP_SINK_COMPLETE, 0);
 
         for (i = 0; i < priv->child_count; i++) {
                 ret = syncop_removexattr (priv->children[i], &loc,
-                                          PUMP_SOURCE_COMPLETE);
-                if (ret)
+                                          PUMP_SOURCE_COMPLETE, 0);
+                if (ret) {
                         gf_log (this->name, GF_LOG_DEBUG, "removexattr "
-                                "failed with %s", strerror (errno));
+                                "failed with %s", strerror (-ret));
+                }
         }
 
         loc_wipe (&loc);
@@ -603,6 +727,7 @@ pump_lookup_sink (loc_t *loc)
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Lookup on sink child failed");
+                ret = -1;
                 goto out;
         }
 
@@ -704,7 +829,7 @@ pump_start (call_frame_t *pump_frame, xlator_t *this)
 	priv = this->private;
         pump_priv = priv->pump_private;
 
-        afr_set_lk_owner (pump_frame, this);
+        afr_set_lk_owner (pump_frame, this, pump_frame->root);
 	pump_pid = (uint64_t) (unsigned long)pump_frame->root;
 
 	ret = synctask_new (pump_priv->env, pump_task,
@@ -753,7 +878,7 @@ pump_cmd_start_setxattr_cbk (call_frame_t *frame,
                              void *cookie,
                              xlator_t *this,
                              int32_t op_ret,
-                             int32_t op_errno)
+                             int32_t op_errno, dict_t *xdata)
 
 {
         call_frame_t *prev = NULL;
@@ -846,7 +971,7 @@ pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
 		    PUMP_SINK_CHILD(this)->fops->setxattr,
 		    &loc,
 		    dict,
-		    0);
+		    0, NULL);
 
         ret = 0;
 
@@ -880,7 +1005,7 @@ pump_cmd_start_getxattr_cbk (call_frame_t *frame,
                              xlator_t *this,
                              int32_t op_ret,
                              int32_t op_errno,
-                             dict_t *dict)
+                             dict_t *dict, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         char *path = NULL;
@@ -947,6 +1072,7 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         uint64_t number_files = 0;
 
         char filename[PATH_MAX];
+        char summary[PATH_MAX+256];
         char *dict_str = NULL;
 
         int32_t op_ret = 0;
@@ -975,12 +1101,15 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         }
 
         if (pump_priv->pump_finished) {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"        Migration complete ",
-                  number_files);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64, number_files);
         } else {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"       Current file= %s ",
-                  number_files, filename);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64":current_file=%s",
+                          number_files, filename);
         }
+        snprintf (dict_str, PATH_MAX+256, "status=%d:%s",
+                  (pump_priv->pump_finished)?1:0, summary);
 
         dict = dict_new ();
 
@@ -996,13 +1125,12 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
 
 out:
 
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
 
         if (dict)
                 dict_unref (dict);
 
-        if (dict_str)
-                GF_FREE (dict_str);
+        GF_FREE (dict_str);
 
         return 0;
 }
@@ -1051,7 +1179,7 @@ pump_execute_start (call_frame_t *frame, xlator_t *this)
 		    PUMP_SOURCE_CHILD(this),
 		    PUMP_SOURCE_CHILD(this)->fops->getxattr,
 		    &loc,
-		    PUMP_PATH);
+		    PUMP_PATH, NULL);
 
         ret = 0;
 
@@ -1069,7 +1197,7 @@ static int
 pump_cleanup_helper (void *data) {
         call_frame_t *frame = data;
 
-        pump_xattr_cleaner (frame, 0, frame->this, 0, 0);
+        pump_xattr_cleaner (frame, 0, frame->this, 0, 0, NULL);
 
         return 0;
 }
@@ -1151,7 +1279,7 @@ pump_execute_abort (call_frame_t *frame, xlator_t *this)
         } else {
                 pump_priv->cleaner = fop_setxattr_cbk_stub (frame,
                                                             pump_xattr_cleaner,
-                                                            0, 0);
+                                                            0, 0, NULL);
         }
 
         return 0;
@@ -1277,143 +1405,24 @@ out:
 
 }
 
-struct _xattr_key {
-        char *key;
-        struct list_head list;
-};
-
-static void
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
-{
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
-
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
-
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return;
-
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
-
-                list_add_tail (&xkey->list, list);
-        }
-}
-
-static void
-__filter_xattrs (dict_t *dict)
-{
-        struct list_head keys;
-
-        struct _xattr_key *key;
-        struct _xattr_key *tmp;
-
-        INIT_LIST_HEAD (&keys);
-
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
-
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
-
-                list_del_init (&key->list);
-
-                GF_FREE (key);
-        }
-}
-
-int32_t
-pump_getxattr_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict)
-{
-	afr_private_t   *priv           = NULL;
-	afr_local_t     *local          = NULL;
-	xlator_t        **children      = NULL;
-	int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children = NULL;
-
-
-	priv     = this->private;
-	children = priv->children;
-
-	local = frame->local;
-
-        read_child = (long) cookie;
-
-	if (op_ret == -1) {
-		last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
-				   (void *) (long) read_child,
-				   children[next_call_child],
-				   children[next_call_child]->fops->getxattr,
-				   &local->loc,
-				   local->cont.getxattr.name);
-	}
-
-out:
-	if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
-
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
-	}
-
-	return 0;
-}
-
-int32_t
-pump_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name)
+int
+pump_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	       const char *name, dict_t *xdata)
 {
-	afr_private_t *   priv       = NULL;
-	xlator_t **       children   = NULL;
-	int               call_child = 0;
-	afr_local_t       *local     = NULL;
-	int32_t           ret     = -1;
-	int32_t           op_errno   = 0;
-        uint64_t          read_child = 0;
-
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+	afr_private_t *priv = NULL;
+	int op_errno = 0;
+	int ret = 0;
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+	priv = this->private;
 
-	children = priv->children;
         if (!priv->use_afr_in_pump) {
                 STACK_WIND (frame, default_getxattr_cbk,
                             FIRST_CHILD (this),
                             (FIRST_CHILD (this))->fops->getxattr,
-                            loc, name);
+                            loc, name, xdata);
                 return 0;
         }
 
-
-	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-	local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
         if (name) {
                 if (!strncmp (name, AFR_XATTR_PREFIX,
                               strlen (AFR_XATTR_PREFIX))) {
@@ -1431,167 +1440,12 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        local->fresh_children = GF_CALLOC (priv->child_count,
-                                          sizeof (*local->fresh_children),
-                                          gf_afr_mt_int32_t);
-        if (!local->fresh_children) {
-                ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-	loc_copy (&local->loc, loc);
-	if (name)
-	  local->cont.getxattr.name       = gf_strdup (name);
-
-	STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name);
+	afr_getxattr (frame, this, loc, name, xdata);
 
 	ret = 0;
 out:
 	if (ret < 0)
-		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
-	return 0;
-}
-
-static int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *   local = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno);
-	}
-	return 0;
-}
-
-static int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int need_unwind = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->child_count) {
-				need_unwind = 1;
-			}
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-
-	return 0;
-}
-
-static int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->setxattr,
-					   &local->loc,
-					   local->cont.setxattr.dict,
-					   local->cont.setxattr.flags);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-static int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t * local = frame->local;
-
-	local->transaction.unwind (frame, this);
-
-	AFR_STACK_DESTROY (frame);
-
-	return 0;
-}
-
-int32_t
-pump_setxattr_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
+		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
@@ -1609,124 +1463,85 @@ pump_command_reply (call_frame_t *frame, xlator_t *this)
                 gf_log (this->name, GF_LOG_INFO,
                         "Command succeeded");
 
-        dict_unref (local->dict);
-
         AFR_STACK_UNWIND (setxattr,
                           frame,
                           local->op_ret,
-                          local->op_errno);
+                          local->op_errno, NULL);
 
         return 0;
 }
 
 int
-pump_parse_command (call_frame_t *frame, xlator_t *this,
-                    afr_local_t *local, dict_t *dict)
+pump_parse_command (call_frame_t *frame, xlator_t *this, dict_t *dict,
+		    int *op_errno_p)
 {
-
+	afr_local_t *local = NULL;
         int ret = -1;
+	int op_errno = 0;
 
         if (pump_command_start (this, dict)) {
-                frame->local = local;
+                local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_start (frame, this);
 
         } else if (pump_command_pause (this, dict)) {
-                frame->local = local;
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_pause (frame, this);
 
         } else if (pump_command_abort (this, dict)) {
-                frame->local = local;
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_abort (frame, this);
 
         } else if (pump_command_commit (this, dict)) {
-                frame->local = local;
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_commit (frame, this);
         }
+out:
+	if (op_errno_p)
+		*op_errno_p = op_errno;
         return ret;
 }
 
 int
-pump_setxattr (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *dict, int32_t flags)
+pump_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+	       int32_t flags, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
-        data_pair_t   * trav  = NULL;
+	afr_private_t *priv = NULL;
 	int ret = -1;
 	int op_errno = 0;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict,
-                                   trav, op_errno, out);
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict, op_errno, out);
 
 	priv = this->private;
         if (!priv->use_afr_in_pump) {
                 STACK_WIND (frame, default_setxattr_cbk,
                             FIRST_CHILD (this),
                             (FIRST_CHILD (this))->fops->setxattr,
-                            loc, dict, flags);
+                            loc, dict, flags, xdata);
                 return 0;
         }
 
-
-	AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-
-	ret = afr_local_init (local, priv, &op_errno);
-	if (ret < 0) {
-                afr_local_cleanup (local, this);
-		goto out;
-        }
-
-        ret = pump_parse_command (frame, this,
-                                  local, dict);
-        if (ret >= 0) {
-                ret = 0;
+        ret = pump_parse_command (frame, this, dict, &op_errno);
+        if (ret >= 0)
                 goto out;
-        }
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-                op_errno = ENOMEM;
-                ret = -1;
-                afr_local_cleanup (local, this);
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	local->op_ret = -1;
-
-	local->cont.setxattr.dict  = dict_ref (dict);
-	local->cont.setxattr.flags = flags;
-
-	local->transaction.fop    = afr_setxattr_wind;
-	local->transaction.done   = afr_setxattr_done;
-	local->transaction.unwind = afr_setxattr_unwind;
 
-	loc_copy (&local->loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.start   = LLONG_MAX - 1;
-	local->transaction.len     = 0;
-
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+	afr_setxattr (frame, this, loc, dict, flags, xdata);
 
 	ret = 0;
 out:
 	if (ret < 0) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno);
+		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 	}
 
 	return 0;
@@ -1760,7 +1575,7 @@ static int32_t
 pump_truncate (call_frame_t *frame,
                xlator_t *this,
                loc_t *loc,
-               off_t offset)
+               off_t offset, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1770,11 +1585,11 @@ pump_truncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->truncate,
                             loc,
-                            offset);
+                            offset, xdata);
                 return 0;
         }
 
-        afr_truncate (frame, this, loc, offset);
+        afr_truncate (frame, this, loc, offset, xdata);
         return 0;
 }
 
@@ -1783,7 +1598,7 @@ static int32_t
 pump_ftruncate (call_frame_t *frame,
                 xlator_t *this,
                 fd_t *fd,
-                off_t offset)
+                off_t offset, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1793,11 +1608,11 @@ pump_ftruncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->ftruncate,
                             fd,
-                            offset);
+                            offset, xdata);
                 return 0;
         }
 
-        afr_ftruncate (frame, this, fd, offset);
+        afr_ftruncate (frame, this, fd, offset, xdata);
         return 0;
 }
 
@@ -1806,7 +1621,7 @@ pump_ftruncate (call_frame_t *frame,
 
 int
 pump_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, dict_t *parms)
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1814,10 +1629,10 @@ pump_mknod (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mknod_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, parms);
+                            loc, mode, rdev, umask, xdata);
                 return 0;
         }
-        afr_mknod (frame, this, loc, mode, rdev, parms);
+        afr_mknod (frame, this, loc, mode, rdev, umask, xdata);
         return 0;
 
 }
@@ -1826,7 +1641,7 @@ pump_mknod (call_frame_t *frame, xlator_t *this,
 
 int
 pump_mkdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dict_t *params)
+            loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1834,10 +1649,10 @@ pump_mkdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mkdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, params);
+                            loc, mode, umask, xdata);
                 return 0;
         }
-        afr_mkdir (frame, this, loc, mode, params);
+        afr_mkdir (frame, this, loc, mode, umask, xdata);
         return 0;
 
 }
@@ -1846,7 +1661,7 @@ pump_mkdir (call_frame_t *frame, xlator_t *this,
 static int32_t
 pump_unlink (call_frame_t *frame,
              xlator_t *this,
-             loc_t *loc)
+             loc_t *loc, int xflag, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1855,10 +1670,10 @@ pump_unlink (call_frame_t *frame,
                             default_unlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->unlink,
-                            loc);
+                            loc, xflag, xdata);
                 return 0;
         }
-        afr_unlink (frame, this, loc);
+        afr_unlink (frame, this, loc, xflag, xdata);
         return 0;
 
 }
@@ -1866,7 +1681,7 @@ pump_unlink (call_frame_t *frame,
 
 static int
 pump_rmdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int flags)
+            loc_t *loc, int flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 
@@ -1876,11 +1691,11 @@ pump_rmdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_rmdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags);
+                            loc, flags, xdata);
                 return 0;
         }
 
-        afr_rmdir (frame, this, loc, flags);
+        afr_rmdir (frame, this, loc, flags, xdata);
         return 0;
 
 }
@@ -1889,7 +1704,7 @@ pump_rmdir (call_frame_t *frame, xlator_t *this,
 
 int
 pump_symlink (call_frame_t *frame, xlator_t *this,
-              const char *linkpath, loc_t *loc, dict_t *params)
+              const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1897,10 +1712,10 @@ pump_symlink (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_symlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, params);
+                            linkpath, loc, umask, xdata);
                 return 0;
         }
-        afr_symlink (frame, this, linkpath, loc, params);
+        afr_symlink (frame, this, linkpath, loc, umask, xdata);
         return 0;
 
 }
@@ -1910,7 +1725,7 @@ static int32_t
 pump_rename (call_frame_t *frame,
              xlator_t *this,
              loc_t *oldloc,
-             loc_t *newloc)
+             loc_t *newloc, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1919,10 +1734,10 @@ pump_rename (call_frame_t *frame,
                             default_rename_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
                 return 0;
         }
-        afr_rename (frame, this, oldloc, newloc);
+        afr_rename (frame, this, oldloc, newloc, xdata);
         return 0;
 
 }
@@ -1932,7 +1747,7 @@ static int32_t
 pump_link (call_frame_t *frame,
            xlator_t *this,
            loc_t *oldloc,
-           loc_t *newloc)
+           loc_t *newloc, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1941,10 +1756,10 @@ pump_link (call_frame_t *frame,
                             default_link_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
                 return 0;
         }
-        afr_link (frame, this, oldloc, newloc);
+        afr_link (frame, this, oldloc, newloc, xdata);
         return 0;
 
 }
@@ -1953,7 +1768,7 @@ pump_link (call_frame_t *frame,
 static int32_t
 pump_create (call_frame_t *frame, xlator_t *this,
              loc_t *loc, int32_t flags, mode_t mode,
-             fd_t *fd, dict_t *params)
+             mode_t umask, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1961,10 +1776,10 @@ pump_create (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_create_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->create,
-                            loc, flags, mode, fd, params);
+                            loc, flags, mode, umask, fd, xdata);
                 return 0;
         }
-        afr_create (frame, this, loc, flags, mode, fd, params);
+        afr_create (frame, this, loc, flags, mode, umask, fd, xdata);
         return 0;
 
 }
@@ -1974,8 +1789,7 @@ static int32_t
 pump_open (call_frame_t *frame,
            xlator_t *this,
            loc_t *loc,
-           int32_t flags, fd_t *fd,
-           int32_t wbflags)
+           int32_t flags, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1984,10 +1798,10 @@ pump_open (call_frame_t *frame,
                             default_open_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->open,
-                            loc, flags, fd, wbflags);
+                            loc, flags, fd, xdata);
                 return 0;
         }
-        afr_open (frame, this, loc, flags, fd, wbflags);
+        afr_open (frame, this, loc, flags, fd, xdata);
         return 0;
 
 }
@@ -2000,7 +1814,7 @@ pump_writev (call_frame_t *frame,
              struct iovec *vector,
              int32_t count,
              off_t off, uint32_t flags,
-             struct iobref *iobref)
+             struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2013,19 +1827,19 @@ pump_writev (call_frame_t *frame,
                             vector,
                             count,
                             off, flags,
-                            iobref);
+                            iobref, xdata);
                 return 0;
         }
-        afr_writev (frame, this, fd, vector, count, off, flags, iobref);
-        return 0;
 
+        afr_writev (frame, this, fd, vector, count, off, flags, iobref, xdata);
+        return 0;
 }
 
 
 static int32_t
 pump_flush (call_frame_t *frame,
             xlator_t *this,
-            fd_t *fd)
+            fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2034,10 +1848,10 @@ pump_flush (call_frame_t *frame,
                             default_flush_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->flush,
-                            fd);
+                            fd, xdata);
                 return 0;
         }
-        afr_flush (frame, this, fd);
+        afr_flush (frame, this, fd, xdata);
         return 0;
 
 }
@@ -2047,7 +1861,7 @@ static int32_t
 pump_fsync (call_frame_t *frame,
             xlator_t *this,
             fd_t *fd,
-            int32_t flags)
+            int32_t flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2057,10 +1871,10 @@ pump_fsync (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsync,
                             fd,
-                            flags);
+                            flags, xdata);
                 return 0;
         }
-        afr_fsync (frame, this, fd, flags);
+        afr_fsync (frame, this, fd, flags, xdata);
         return 0;
 
 }
@@ -2069,7 +1883,7 @@ pump_fsync (call_frame_t *frame,
 static int32_t
 pump_opendir (call_frame_t *frame,
               xlator_t *this,
-              loc_t *loc, fd_t *fd)
+              loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2078,10 +1892,10 @@ pump_opendir (call_frame_t *frame,
                             default_opendir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->opendir,
-                            loc, fd);
+                            loc, fd, xdata);
                 return 0;
         }
-        afr_opendir (frame, this, loc, fd);
+        afr_opendir (frame, this, loc, fd, xdata);
         return 0;
 
 }
@@ -2091,7 +1905,7 @@ static int32_t
 pump_fsyncdir (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
-               int32_t flags)
+               int32_t flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2101,10 +1915,10 @@ pump_fsyncdir (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsyncdir,
                             fd,
-                            flags);
+                            flags, xdata);
                 return 0;
         }
-        afr_fsyncdir (frame, this, fd, flags);
+        afr_fsyncdir (frame, this, fd, flags, xdata);
         return 0;
 
 }
@@ -2115,7 +1929,7 @@ pump_xattrop (call_frame_t *frame,
               xlator_t *this,
               loc_t *loc,
               gf_xattrop_flags_t flags,
-              dict_t *dict)
+              dict_t *dict, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2126,10 +1940,10 @@ pump_xattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->xattrop,
                             loc,
                             flags,
-                            dict);
+                            dict, xdata);
                 return 0;
         }
-        afr_xattrop (frame, this, loc, flags, dict);
+        afr_xattrop (frame, this, loc, flags, dict, xdata);
         return 0;
 
 }
@@ -2139,7 +1953,7 @@ pump_fxattrop (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
                gf_xattrop_flags_t flags,
-               dict_t *dict)
+               dict_t *dict, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2150,10 +1964,10 @@ pump_fxattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->fxattrop,
                             fd,
                             flags,
-                            dict);
+                            dict, xdata);
                 return 0;
         }
-        afr_fxattrop (frame, this, fd, flags, dict);
+        afr_fxattrop (frame, this, fd, flags, dict, xdata);
         return 0;
 
 }
@@ -2163,7 +1977,7 @@ static int32_t
 pump_removexattr (call_frame_t *frame,
                   xlator_t *this,
                   loc_t *loc,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
         afr_private_t *priv     = NULL;
         int            op_errno = -1;
@@ -2181,14 +1995,14 @@ pump_removexattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->removexattr,
                             loc,
-                            name);
+                            name, xdata);
                 return 0;
         }
-        afr_removexattr (frame, this, loc, name);
+        afr_removexattr (frame, this, loc, name, xdata);
 
  out:
         if (op_errno)
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno);
+                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         return 0;
 
 }
@@ -2200,7 +2014,7 @@ pump_readdir (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               size_t size,
-              off_t off)
+              off_t off, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2209,10 +2023,10 @@ pump_readdir (call_frame_t *frame,
                             default_readdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off);
+                            fd, size, off, xdata);
                 return 0;
         }
-        afr_readdir (frame, this, fd, size, off);
+        afr_readdir (frame, this, fd, size, off, xdata);
         return 0;
 
 }
@@ -2263,13 +2077,24 @@ pump_release (xlator_t *this,
 
 }
 
+static int32_t
+pump_forget (xlator_t *this, inode_t *inode)
+{
+        afr_private_t  *priv  = NULL;
+
+        priv = this->private;
+        if (priv->use_afr_in_pump)
+                afr_forget (this, inode);
+
+        return 0;
+}
 
 static int32_t
 pump_setattr (call_frame_t *frame,
               xlator_t *this,
               loc_t *loc,
               struct iatt *stbuf,
-              int32_t valid)
+              int32_t valid, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2278,10 +2103,10 @@ pump_setattr (call_frame_t *frame,
                             default_setattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, xdata);
                 return 0;
         }
-        afr_setattr (frame, this, loc, stbuf, valid);
+        afr_setattr (frame, this, loc, stbuf, valid, xdata);
         return 0;
 
 }
@@ -2292,7 +2117,7 @@ pump_fsetattr (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
                struct iatt *stbuf,
-               int32_t valid)
+               int32_t valid, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2301,10 +2126,10 @@ pump_fsetattr (call_frame_t *frame,
                             default_fsetattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid);
+                            fd, stbuf, valid, xdata);
                 return 0;
         }
-        afr_fsetattr (frame, this, fd, stbuf, valid);
+        afr_fsetattr (frame, this, fd, stbuf, valid, xdata);
         return 0;
 
 }
@@ -2383,7 +2208,7 @@ int32_t
 init (xlator_t *this)
 {
 	afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv = NULL;
+        pump_private_t *pump_priv   = NULL;
 	int             child_count = 0;
 	xlator_list_t * trav        = NULL;
 	int             i           = 0;
@@ -2404,17 +2229,11 @@ init (xlator_t *this)
 			"Volume is dangling.");
 	}
 
-	this->private = GF_CALLOC (1, sizeof (afr_private_t),
-                                   gf_afr_mt_afr_private_t);
-        if (!this->private)
+	priv = GF_CALLOC (1, sizeof (afr_private_t), gf_afr_mt_afr_private_t);
+        if (!priv)
                 goto out;
 
-	priv = this->private;
         LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->read_child_lock);
-        //lock recovery is not done in afr
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->saved_fds);
 
         child_count = xlator_subvolume_count (this);
         if (child_count != 2) {
@@ -2433,14 +2252,13 @@ init (xlator_t *this)
 	priv->metadata_self_heal = 1;
 	priv->entry_self_heal    = 1;
 
-        priv->data_self_heal_algorithm = "";
-
         priv->data_self_heal_window_size = 16;
 
 	priv->data_change_log     = 1;
 	priv->metadata_change_log = 1;
 	priv->entry_change_log    = 1;
         priv->use_afr_in_pump = 1;
+        priv->sh_readdir_size = 65536;
 
 	/* Locking options */
 
@@ -2449,9 +2267,6 @@ init (xlator_t *this)
            and the sink.
         */
 
-	priv->strict_readdir = _gf_false;
-
-	priv->wait_count = 1;
 	priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
                                  gf_afr_mt_char);
 	if (!priv->child_up) {
@@ -2485,7 +2300,8 @@ init (xlator_t *this)
 	while (i < child_count) {
 		priv->children[i] = trav->xlator;
 
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
+                                   AFR_XATTR_PREFIX,
                                    trav->xlator->name);
                 if (-1 == ret) {
                         gf_log (this->name, GF_LOG_ERROR,
@@ -2498,7 +2314,12 @@ init (xlator_t *this)
 		i++;
 	}
 
-        priv->first_lookup = 1;
+        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
+        if (-1 == ret) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
         priv->root_inode = NULL;
 
         priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
@@ -2523,13 +2344,12 @@ init (xlator_t *this)
         pump_priv->resume_path = GF_CALLOC (1, PATH_MAX,
                                             gf_afr_mt_char);
         if (!pump_priv->resume_path) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
+                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
                 ret = -1;
                 goto out;
         }
 
-	pump_priv->env = syncenv_new (0);
+	pump_priv->env = this->ctx->env;
         if (!pump_priv->env) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Could not create new sync-environment");
@@ -2547,11 +2367,32 @@ init (xlator_t *this)
         }
 
 	priv->pump_private = pump_priv;
+        pump_priv = NULL;
+
+        this->private = priv;
+        priv = NULL;
 
         pump_change_state (this, PUMP_STATE_ABORT);
 
 	ret = 0;
 out:
+
+        if (pump_priv) {
+                GF_FREE (pump_priv->resume_path);
+                LOCK_DESTROY (&pump_priv->resume_path_lock);
+                LOCK_DESTROY (&pump_priv->pump_state_lock);
+                GF_FREE (pump_priv);
+        }
+
+        if (priv) {
+                GF_FREE (priv->child_up);
+                GF_FREE (priv->children);
+                GF_FREE (priv->pending_key);
+                GF_FREE (priv->last_event);
+                LOCK_DESTROY (&priv->lock);
+                GF_FREE (priv);
+        }
+
 	return ret;
 }
 
@@ -2570,9 +2411,6 @@ fini (xlator_t *this)
         if (!pump_priv)
                 goto afr_priv;
 
-        if (pump_priv->env)
-                syncenv_destroy (pump_priv->env);
-
         GF_FREE (pump_priv->resume_path);
         LOCK_DESTROY (&pump_priv->resume_path_lock);
         LOCK_DESTROY (&pump_priv->pump_state_lock);
@@ -2627,6 +2465,7 @@ struct xlator_dumpops dumpops = {
 struct xlator_cbks cbks = {
 	.release     = pump_release,
 	.releasedir  = pump_releasedir,
+        .forget      = pump_forget,
 };
 
 struct volume_options options[] = {
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
index 4213a635e..9d0b6db6a 100644
--- a/xlators/cluster/afr/src/pump.h
+++ b/xlators/cluster/afr/src/pump.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __PUMP_H__
@@ -84,4 +75,7 @@ pump_command_status (xlator_t *this, dict_t *dict);
 int
 pump_execute_status (call_frame_t *frame, xlator_t *this);
 
+int
+pump_command_reply (call_frame_t *frame, xlator_t *this);
+
 #endif /* __PUMP_H__ */
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index e35058d65..174bea841 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -4,7 +4,7 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
 	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
-	dht-common.c dht-inode-write.c dht-inode-read.c \
+	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
 	$(top_builddir)/xlators/lib/src/libxlator.c
 
 dht_la_SOURCES = $(dht_common_source) dht.c
@@ -12,22 +12,23 @@ dht_la_SOURCES = $(dht_common_source) dht.c
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
 
-dht_la_LDFLAGS = -module -avoidversion
+dht_la_LDFLAGS = -module -avoid-version
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoidversion
+nufa_la_LDFLAGS = -module -avoid-version
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoidversion
+switch_la_LDFLAGS = -module -avoid-version
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = dht-common.h dht-mem-types.h \
 	$(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/lib/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 CLEANFILES =
 
 uninstall-local:
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 2bff9c87b..3868fc38f 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -31,17 +22,18 @@
 #include "dht-common.h"
 #include "defaults.h"
 #include "byte-order.h"
+#include "glusterfs-acl.h"
 
 #include <sys/time.h>
 #include <libgen.h>
 
-void
+int
 dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t  *dst  = NULL;
         int64_t *ptr  = 0, *size = NULL;
         int32_t  ret  = -1;
-        data_pair_t  *data_pair = NULL;
+        data_t  *dict_data = NULL;
 
         dst = data;
 
@@ -53,32 +45,37 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         if (size == NULL) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "memory allocation failed");
-                                return;
+                                return -1;
                         }
                         ret = dict_set_bin (dst, key, size, sizeof (int64_t));
                         if (ret < 0) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "dht aggregate dict set failed");
                                 GF_FREE (size);
-                                return;
+                                return -1;
                         }
                 }
 
                 ptr = data_to_bin (value);
                 if (ptr == NULL) {
                         gf_log ("dht", GF_LOG_WARNING, "data to bin failed");
-                        return;
+                        return -1;
                 }
 
                 *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+                ret = gf_get_min_stime (THIS, dst, key, value);
+                if (ret < 0)
+                        return ret;
         } else {
                 /* compare user xattrs only */
                 if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &data_pair); 
-                        if (!ret && data_pair && value) {
-                                ret = is_data_equal (data_pair->value, value);
+                        ret = dict_lookup (dst, key, &dict_data);
+                        if (!ret && dict_data && value) {
+                                ret = is_data_equal (dict_data, value);
                                 if (!ret)
-                                        gf_log ("dht", GF_LOG_WARNING,
+                                        gf_log ("dht", GF_LOG_DEBUG,
                                                 "xattr mismatch for %s", key);
                         }
                 }
@@ -87,7 +84,7 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         gf_log ("dht", GF_LOG_WARNING, "xattr dict set failed");
         }
 
-        return;
+        return 0;
 }
 
 
@@ -114,7 +111,7 @@ out:
 int
 dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
                          xlator_t *this,
-                         int op_ret, int op_errno)
+                         int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
@@ -134,7 +131,10 @@ dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
                 ret = dht_layout_set (this, local->inode, layout);
         }
 
-        WIPE (&local->postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           &local->postparent, 1);
+        }
 
         DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
 
@@ -154,9 +154,11 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
         int              op_errno = 0;
         int              ret = -1;
         dht_layout_t    *layout = NULL;
+        dht_conf_t      *conf = NULL;
 
         local = discover_frame->local;
         layout = local->layout;
+        conf = this->private;
 
         LOCK(&discover_frame->lock);
         {
@@ -190,16 +192,23 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
                 }
         } else {
                 ret = dht_layout_normalize (this, &local->loc, layout);
-
-                if (ret != 0) {
+                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+                        /* either the layout is incorrect or the directory is
+                         * not found even in one subvolume.
+                         */
                         gf_log (this->name, GF_LOG_DEBUG,
-                                "normalizing failed on %s",
-                                local->loc.path);
-                        op_errno = EINVAL;
-                        goto out;
+                                "normalizing failed on %s "
+                                "(overlaps/holes present: %s, "
+                                "ENOENT errors: %d)", local->loc.path,
+                                (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
+                        if ((ret > 0) && (ret == conf->subvolume_cnt)) {
+                                op_errno = ESTALE;
+                                goto out;
+                        }
                 }
 
-                dht_layout_set (this, local->inode, layout);
+                if (local->inode)
+                        dht_layout_set (this, local->inode, layout);
         }
 
         DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
@@ -228,6 +237,7 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           is_dir                  = 0;
         int           is_linkfile             = 0;
         int           attempt_unwind          = 0;
+        dht_conf_t   *conf                    = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -237,6 +247,7 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local = frame->local;
         prev  = cookie;
+        conf = this->private;
 
         layout = local->layout;
 
@@ -271,7 +282,8 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto unlock;
                 }
 
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
                 is_dir = check_is_dir (inode, stbuf, xattr);
 
                 if (is_dir) {
@@ -330,23 +342,20 @@ dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
         int          i = 0;
         call_frame_t *discover_frame = NULL;
 
-
         conf = this->private;
         local = frame->local;
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht", 4 * 4);
+        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht' key",
-                        loc->path);
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->xattr_name);
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht.linkto", 256);
+        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        loc->path);
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->link_xattr_name);
 
         call_cnt        = conf->subvolume_cnt;
         local->call_cnt = call_cnt;
@@ -380,7 +389,8 @@ dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
         return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
 
         return 0;
 }
@@ -431,7 +441,7 @@ dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         op_ret, op_errno, xattr);
 
                 if (op_ret == -1) {
-                        local->op_errno = ENOENT;
+                        local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "lookup of %s on %s returned error (%s)",
                                 local->loc.path, prev->this->name,
@@ -490,6 +500,11 @@ unlock:
                         dht_layout_set (this, local->inode, layout);
                 }
 
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                                   local->inode, &local->stbuf, local->xattr,
@@ -521,6 +536,8 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           ret  = -1;
         int           is_dir = 0;
         int           is_linkfile = 0;
+        call_frame_t *copy          = NULL;
+        dht_local_t  *copy_local    = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, err);
@@ -579,7 +596,8 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 layout = local->layout;
 
                 is_dir = check_is_dir (inode, stbuf, xattr);
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
 
                 if (is_linkfile) {
                         gf_log (this->name, GF_LOG_INFO,
@@ -591,6 +609,23 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 if (is_dir) {
+                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
+                        if (ret >= 0) {
+                                if (is_greater_time(local->stbuf.ia_ctime,
+                                                    local->stbuf.ia_ctime_nsec,
+                                                    stbuf->ia_ctime,
+                                                    stbuf->ia_ctime_nsec)) {
+                                        local->prebuf.ia_gid = stbuf->ia_gid;
+                                        local->prebuf.ia_uid = stbuf->ia_uid;
+                                }
+                        }
+                        if (local->stbuf.ia_type != IA_INVAL)
+                        {
+                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                                    (local->stbuf.ia_uid != stbuf->ia_uid)) {
+                                        local->need_selfheal = 1;
+                                }
+                        }
                         ret = dht_layout_dir_mismatch (this, layout,
                                                        prev->this, &local->loc,
                                                        xattr);
@@ -629,7 +664,28 @@ out:
                     && (conf && conf->unhashed_sticky_bit)) {
                         local->stbuf.ia_prot.sticky = 1;
                 }
-                if (local->layout_mismatch) {
+                if (local->need_selfheal) {
+                        local->need_selfheal = 0;
+                        uuid_copy (local->gfid, local->stbuf.ia_gfid);
+                        local->stbuf.ia_gid = local->prebuf.ia_gid;
+                        local->stbuf.ia_uid = local->prebuf.ia_uid;
+                        copy = create_frame (this, this->ctx->pool);
+                        if (copy) {
+                                copy_local = dht_local_init (copy, &local->loc,
+                                                             NULL, 0);
+                                if (!copy_local)
+                                        goto cont;
+                                copy_local->stbuf = local->stbuf;
+                                copy->local = copy_local;
+                                FRAME_SU_DO (copy, dht_local_t);
+                                ret = synctask_new (this->ctx->env,
+                                                    dht_dir_attr_heal,
+                                                    dht_dir_attr_heal_done,
+                                                    copy, copy);
+                        }
+                }
+cont:
+		if (local->layout_mismatch) {
                         /* Found layout mismatch in the directory, need to
                            fix this in the inode context */
                         dht_layout_unref (this, local->layout);
@@ -655,7 +711,10 @@ out:
                         local->op_errno = ESTALE;
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
@@ -673,7 +732,8 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this,
                                 int32_t op_ret, int32_t op_errno,
                                 inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent)
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -690,7 +750,7 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
         cached_subvol = local->cached_subvol;
         conf = this->private;
 
-        ret = dht_layout_preset (this, local->cached_subvol, inode);
+        ret = dht_layout_preset (this, local->cached_subvol, local->loc.inode);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "failed to set layout for subvolume %s",
@@ -706,8 +766,14 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 local->stbuf.ia_prot.sticky = 1;
         }
 
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+
 unwind:
-        WIPE (&local->postparent);
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 
         DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
         DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
@@ -780,7 +846,10 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                                                   "<nil>"));
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
@@ -810,7 +879,10 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                         local->op_errno = EINVAL;
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
@@ -826,7 +898,7 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                 hashed_subvol->name);
 
         ret = dht_linkfile_create (frame,
-                                   dht_lookup_linkfile_create_cbk,
+                                   dht_lookup_linkfile_create_cbk, this,
                                    cached_subvol, hashed_subvol, &local->loc);
 
         return ret;
@@ -836,7 +908,8 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
 int
 dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
-                       struct iatt *preparent, struct iatt *postparent)
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
         int  this_call_cnt = 0;
 
@@ -863,8 +936,9 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         xlator_t     *subvol        = NULL;
         loc_t        *loc           = NULL;
         xlator_t     *link_subvol   = NULL;
-        int           ret = -1;
-        int32_t       fd_count = 0;
+        int           ret           = -1;
+        int32_t       fd_count      = 0;
+        dht_conf_t   *conf          = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -874,6 +948,7 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local  = frame->local;
         loc    = &local->loc;
+        conf   = this->private;
 
         prev   = cookie;
         subvol = prev->this;
@@ -895,7 +970,8 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 loc->path, prev->this->name);
                 }
 
-                is_linkfile = check_is_linkfile (inode, buf, xattr);
+                is_linkfile = check_is_linkfile (inode, buf, xattr,
+                                                 conf->link_xattr_name);
                 is_dir = check_is_dir (inode, buf, xattr);
 
                 if (is_linkfile) {
@@ -955,7 +1031,7 @@ unlock:
                                 "deleting stale linkfile %s on %s",
                                 loc->path, subvol->name);
                         STACK_WIND (frame, dht_lookup_unlink_cbk,
-                                    subvol, subvol->fops->unlink, loc);
+                                    subvol, subvol->fops->unlink, loc, 0, NULL);
                         return 0;
                 }
         }
@@ -1037,7 +1113,16 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) failed (%s)",
                         local->loc.path, subvol->name, strerror (op_errno));
-                goto err;
+
+                /* If cached subvol returned ENOTCONN, do not do
+                lookup_everywhere. We need to make sure linkfile does not get
+                removed, which can take away the namespace, and subvol is
+                anyways down. */
+
+                if (op_errno != ENOTCONN)
+                        goto err;
+                else
+                        goto unwind;
         }
 
         if (check_is_dir (inode, stbuf, xattr)) {
@@ -1047,7 +1132,7 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 goto err;
         }
 
-        if (check_is_linkfile (inode, stbuf, xattr)) {
+        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) reached link",
                         local->loc.path, subvol->name);
@@ -1075,9 +1160,12 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 op_errno = EINVAL;
         }
 
-unwind:
-        WIPE (postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
+unwind:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
                           postparent);
@@ -1159,7 +1247,6 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         loc_t        *loc           = NULL;
         call_frame_t *prev          = NULL;
         int           ret           = 0;
-        uint64_t      tmp_layout    = 0;
         dht_layout_t *parent_layout = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
@@ -1190,8 +1277,10 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
                 if ((conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) &&
                     (loc->parent)) {
-                        ret = inode_ctx_get (loc->parent, this, &tmp_layout);
-                        parent_layout = (dht_layout_t *)(long)tmp_layout;
+                        ret = dht_inode_ctx_layout_get (loc->parent, this,
+                                                        &parent_layout);
+                        if (ret || !parent_layout)
+                                goto out;
                         if (parent_layout->search_unhashed) {
                                 local->op_errno = ENOENT;
                                 dht_lookup_everywhere (frame, this, loc);
@@ -1220,7 +1309,8 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
 
         if (!is_linkfile) {
                 /* non-directory and not a linkfile */
@@ -1260,7 +1350,10 @@ out:
          * from each of the subvolume. See dht_iatt_merge for reference.
          */
 
-        WIPE (postparent);
+        if (!op_ret && local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
@@ -1269,6 +1362,39 @@ err:
         return 0;
 }
 
+/* For directories, check if acl xattrs have been requested (by the acl xlator),
+ * if not, request for them. These xattrs are needed for dht dir self-heal to
+ * perform proper self-healing of dirs
+ */
+void
+dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+{
+        int     ret = 0;
+
+        GF_ASSERT (inode);
+        GF_ASSERT (xattr_req);
+
+        if (inode->ia_type != IA_IFDIR)
+                return;
+
+        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_ACCESS_XATTR);
+        }
+
+        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_DEFAULT_XATTR);
+        }
+
+        return;
+}
 
 int
 dht_lookup (call_frame_t *frame, xlator_t *this,
@@ -1289,7 +1415,6 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         conf = this->private;
         if (!conf)
@@ -1333,16 +1458,8 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
-        if (!hashed_subvol) {
+        if (!hashed_subvol)
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (!hashed_subvol) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-        }
         local->hashed_subvol = hashed_subvol;
 
         if (is_revalidate (loc)) {
@@ -1372,7 +1489,7 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                  *       revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
 
                 if (IA_ISDIR (local->inode->ia_type)) {
                         local->call_cnt = call_cnt = conf->subvolume_cnt;
@@ -1392,6 +1509,9 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                 ret = dict_set_uint32 (local->xattr_req,
                                        GLUSTERFS_OPEN_FD_COUNT, 4);
 
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+
 		for (i = 0; i < call_cnt; i++) {
 			subvol = layout->list[i].xlator;
 
@@ -1404,16 +1524,19 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         do_fresh_lookup:
                 /* TODO: remove the hard-coding */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       DHT_LINKFILE_KEY, 256);
+                                       conf->link_xattr_name, 256);
 
                 /* need it for self-healing linkfiles which is
                    'in-migration' state */
                 ret = dict_set_uint32 (local->xattr_req,
                                        GLUSTERFS_OPEN_FD_COUNT, 4);
 
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+
                 if (!hashed_subvol) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "no subvolume in layout for path=%s, "
@@ -1447,7 +1570,8 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
@@ -1455,7 +1579,7 @@ err:
 int
 dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev  = NULL;
@@ -1479,14 +1603,18 @@ dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->postparent = *postparent;
                 local->preparent = *preparent;
 
-                WIPE (&local->postparent);
-                WIPE (&local->preparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
         }
 unlock:
         UNLOCK (&frame->lock);
 
         DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
 
         return 0;
 }
@@ -1495,7 +1623,7 @@ unlock:
 int
 dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -1507,7 +1635,8 @@ dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == -1) {
+                if ((op_ret == -1) && !((op_errno == ENOENT) ||
+                                        (op_errno == ENOTCONN))) {
                         local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "subvolume %s returned -1 (%s)",
@@ -1520,7 +1649,7 @@ dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unlock:
         UNLOCK (&frame->lock);
 
-        if (op_ret == -1)
+        if (local->op_ret == -1)
                 goto err;
 
         cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
@@ -1534,53 +1663,19 @@ unlock:
 
         STACK_WIND (frame, dht_unlink_cbk,
                     cached_subvol, cached_subvol->fops->unlink,
-                    &local->loc);
+                    &local->loc, local->flags, NULL);
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL);
-        return 0;
-}
-
-static int
-dht_ufo_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno);
-        }
-
+                          NULL, NULL, NULL);
         return 0;
 }
 
-
 int
 dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int op_ret, int op_errno)
+             int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -1606,7 +1701,8 @@ unlock:
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
         }
 
         return 0;
@@ -1649,125 +1745,232 @@ dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
 }
 
 int
-dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
+                              int op_errno)
 {
-        dht_local_t *local         = NULL;
-        int          ret           = 0;
-        int          flag          = 0;
-        int          this_call_cnt = 0;
-        char        *value_got     = NULL;
-        char  layout_buf[8192]     = {0,};
-        char        *xattr_buf     = NULL;
-        dict_t      *dict          = NULL;
-        int32_t      alloc_len     = 0;
-        int32_t      plen          = 0;
+        int      ret       = -1;
+        char    *value     = NULL;
+        int32_t  plen      = 0;
 
-        local = frame->local;
+        ret = dict_get_str (xattr, local->xsel, &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Subvolume %s returned -1 (%s)", this->name,
+                        strerror (op_errno));
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                goto out;
+        }
 
-        if (op_ret != -1) {
-                ret = dict_get_str (xattr, local->xsel, &value_got);
-                if (!ret) {
-                        alloc_len = strlen (value_got);
+        local->alloc_len += strlen(value);
 
-                        /**
-                         * allocate the buffer:- we allocate 10 bytes extra in
-                         * case we need to append ' Link: ' in the buffer for
-                         * another STACK_WIND
-                         */
+        if (!local->xattr_val) {
+                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
+                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
+                                              gf_common_mt_char);
+                if (!local->xattr_val) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (local->xattr_val) {
+                plen = strlen (local->xattr_val);
+                if (plen) {
+                        /* extra byte(s) for \0 to be safe */
+                        local->alloc_len += (plen + 2);
+                        local->xattr_val = GF_REALLOC (local->xattr_val,
+                                                       local->alloc_len);
                         if (!local->xattr_val) {
-                                alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
-                                local->xattr_val =
-                                        GF_CALLOC (alloc_len,
-                                                   sizeof (char),
-                                                   gf_common_mt_char);
+                                ret = -1;
+                                goto out;
                         }
+                }
 
-                        if (local->xattr_val) {
-                                plen = strlen (local->xattr_val);
-                                if (plen) {
-                                        /* extra byte(s) for \0 to be safe */
-                                        alloc_len += (plen + 2);
-                                        local->xattr_val =
-                                                GF_REALLOC (local->xattr_val,
-                                                            alloc_len);
-                                        if (!local->xattr_val)
-                                                goto out;
-                                }
+                (void) strcat (local->xattr_val, value);
+                (void) strcat (local->xattr_val, " ");
+                local->op_ret = 0;
+        }
 
-                                strcat (local->xattr_val, value_got);
-                        }
-                }
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int
+dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
+                            gf_boolean_t flag)
+{
+        int   ret             = -1;
+        char *xattr_buf       = NULL;
+        char layout_buf[8192] = {0,};
+
+        if (flag)
+                fill_layout_info (local->layout, layout_buf);
+
+        *dict = dict_new ();
+        if (!*dict)
+                goto out;
+
+        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+
+        /* we would need max this many bytes to create xattr string
+         * extra 40 bytes is just an estimated amount of additional
+         * space required as we include translator name and some
+         * spaces, brackets etc. when forming the pathinfo string.
+         *
+         * For node-uuid we just don't have all the pretty formatting,
+         * but since this is a generic routine for pathinfo & node-uuid
+         * we dont have conditional space allocation and try to be
+         * generic
+         */
+        local->alloc_len += (2 * strlen (this->name))
+                + strlen (layout_buf)
+                + 40;
+        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
+                               gf_common_mt_char);
+        if (!xattr_buf)
+                goto out;
+
+        if (XATTR_IS_PATHINFO (local->xsel)) {
+                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
+                                                local->alloc_len, flag,
+                                                layout_buf);
+        } else if (XATTR_IS_NODE_UUID (local->xsel)) {
+                (void) snprintf (xattr_buf, local->alloc_len, "%s",
+                                 local->xattr_val);
+        } else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unknown local->xsel (%s)", local->xsel);
+                GF_FREE (xattr_buf);
+                goto out;
         }
 
+        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
+        if (ret)
+                GF_FREE (xattr_buf);
+        GF_FREE (local->xattr_val);
+
  out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                if (local->layout->cnt > 1) {
-                        /* Set it for directory */
-                        fill_layout_info (local->layout, layout_buf);
-                        flag = 1;
-                }
+        return ret;
+}
 
-                dict = dict_new ();
-
-                /* we would need max this many bytes to create xattr string */
-                alloc_len += (2 * strlen (this->name))
-                          + strlen (layout_buf)
-                          + 40;
-                xattr_buf = GF_CALLOC (alloc_len,
-                                       sizeof (char), gf_common_mt_char);
-
-                if (XATTR_IS_PATHINFO (local->xsel)) {
-                        (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
-                                                        alloc_len, flag,
-                                                        layout_buf);
-                } else if (XATTR_IS_NODE_UUID (local->xsel)) {
-                        (void) snprintf (xattr_buf, alloc_len, "%s",
-                                         local->xattr_val);
-                } else
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unknown local->xsel (%s)", local->xsel);
+int
+dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        int          ret           = 0;
+        dht_local_t *local         = NULL;
+        int          this_call_cnt = 0;
+        dict_t      *dict          = NULL;
 
-                ret = dict_set_dynstr (dict, local->xsel, xattr_buf);
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
 
-                if (local->xattr_val)
-                        GF_FREE (local->xattr_val);
+        local = frame->local;
 
-                DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        LOCK (&frame->lock);
+        {
+                this_call_cnt = --local->call_cnt;
+                if (op_ret < 0) {
+                        if (op_errno != ENOTCONN) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "getxattr err (%s) for dir",
+                                        strerror (op_errno));
+				local->op_ret = -1;
+				local->op_errno = op_errno;
+                        }
 
-                if (dict)
-                        dict_unref (dict);
+                        goto unlock;
+                }
 
-                return 0;
+                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                                    op_errno);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "alloc or fill failure");
         }
+ unlock:
+        UNLOCK (&frame->lock);
 
-        /**
-         * XXX: We will never reach here as call_cnt will always be 1.
-         * But code is kept untouched as we may need it when hashed_subvol
-         * bug is fixed.
-         */
-        if (local->xattr_val)
-                strcat (local->xattr_val, " Link: ");
-        if (local->hashed_subvol) {
-                /* This will happen if there pending */
-                STACK_WIND (frame, dht_vgetxattr_cbk, local->hashed_subvol,
-                            local->hashed_subvol->fops->getxattr,
-                            &local->loc, local->key);
+        if (!is_last_call (this_call_cnt))
+                goto out;
 
-                return 0;
+        /* -- last call: do patch ups -- */
+
+        if (local->op_ret == -1) {
+                goto unwind;
+        }
+
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
+        if (ret)
+                goto unwind;
+
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
+
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
+ out:
+        return 0;
+}
+
+int
+dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        dht_local_t  *local         = NULL;
+        int           ret           = 0;
+        dict_t       *dict          = NULL;
+        call_frame_t *prev          = NULL;
+        gf_boolean_t  flag          = _gf_true;
+
+        local = frame->local;
+        prev = cookie;
+
+        if (op_ret < 0) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_ERROR, "Subvolume %s returned -1 "
+                        "(%s)", prev->this->name, strerror (op_errno));
+                goto unwind;
         }
 
-        gf_log ("this->name", GF_LOG_ERROR, "Unable to find hashed_subvol"
-                " for path %s", local->xattr_val);
+        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                            op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "alloc or fill failure");
+                goto unwind;
+        }
+
+        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
+        if (ret)
+                goto unwind;
+
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
+
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
+                          NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
 
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, dict);
         return 0;
 }
 
 int
 dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         int   ret   = 0;
         char *value = NULL;
@@ -1782,21 +1985,24 @@ dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
         }
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
         return 0;
 }
 
 int
 dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xattr)
+                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
         int             this_call_cnt = 0;
         dht_local_t     *local = NULL;
+        dht_conf_t      *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (frame->local, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
+        conf = this->private;
         local = frame->local;
 
         this_call_cnt = dht_frame_return (frame);
@@ -1804,42 +2010,106 @@ dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!xattr || (op_ret == -1))
                 goto out;
 
-        if (dict_get (xattr, "trusted.glusterfs.dht")) {
-                dict_del (xattr, "trusted.glusterfs.dht");
+        if (dict_get (xattr, conf->xattr_name)) {
+                dict_del (xattr, conf->xattr_name);
+        }
+
+        if (frame->root->pid >= 0 ) {
+                GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+                GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
         }
+
         local->op_ret = 0;
 
         if (!local->xattr) {
                 local->xattr = dict_copy_with_ref (xattr, NULL);
         } else {
-                /* first aggregate everything into xattr and then copy into
-                 * local->xattr. This is required as we want to have
-                 * 'local->xattr' as the proper final dictionary passed above
-                 * distribute xlator.
-                 */
-                dht_aggregate_xattr (xattr, local->xattr);
-                local->xattr = dict_copy (xattr, local->xattr);
+                dht_aggregate_xattr (local->xattr, xattr);
         }
 out:
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno, local->xattr);
+                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+                                  local->xattr, NULL);
         }
         return 0;
 }
 
 int32_t
 dht_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict)
+                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
+dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
+				    xlator_t *this, int op_ret, int op_errno,
+				    dict_t *xattr, dict_t *xdata)
+{
+        int             this_call_cnt = 0;
+        dht_local_t     *local = NULL;
+
+
+        local = frame->local;
+
+	if (op_ret != -1) {
+		if (local->xattr)
+			dict_unref (local->xattr);
+		local->xattr = dict_ref (xattr);
+
+		if (local->xattr_req)
+			dict_unref (local->xattr_req);
+		local->xattr_req = dict_ref (xdata);
+	}
+
+	this_call_cnt = dht_frame_return (frame);
+	if (is_last_call (this_call_cnt)) {
+		DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+				  local->xattr, local->xattr_req);
+	}
+
+	return 0;
+}
+
+
+int
+dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
+				loc_t *loc, const char *key, dict_t *xdata)
+{
+	dht_local_t     *local = NULL;
+	int              i = 0;
+	dht_layout_t    *layout = NULL;
+	int              cnt = 0;
+	xlator_t        *subvol = NULL;
+
+
+	local = frame->local;
+	layout = local->layout;
+
+	cnt = local->call_cnt = layout->cnt;
+
+	local->op_ret = -1;
+	local->op_errno = ENODATA;
+
+	for (i = 0; i < cnt; i++) {
+		subvol = layout->list[i].xlator;
+		STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
+			    subvol, subvol->fops->getxattr,
+			    loc, key, xdata);
+	}
+
+	return 0;
+}
+
+
+int
 dht_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *key)
+              loc_t *loc, const char *key, dict_t *xdata)
+#define DHT_IS_DIR(layout)  (layout->cnt > 1)
 {
+
         xlator_t     *subvol        = NULL;
         xlator_t     *hashed_subvol = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -1855,7 +2125,6 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf   = this->private;
@@ -1883,17 +2152,50 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (key && ((strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)
-                    || strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)) {
+	if (key &&
+	    (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
+		      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
+	    && DHT_IS_DIR(layout)) {
+		dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
+		return 0;
+	}
+
+        /* for file use cached subvolume (obviously!): see if {}
+         * below
+         * for directory:
+         *  wind to all subvolumes and exclude subvolumes which
+         *  return ENOTCONN (in callback)
+         *
+         * NOTE: Don't trust inode here, as that may not be valid
+         *       (until inode_link() happens)
+         */
+        if (key && DHT_IS_DIR(layout) &&
+            (XATTR_IS_PATHINFO (key)
+             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0))) {
+                (void) strncpy (local->xsel, key, 256);
+                cnt = local->call_cnt = layout->cnt;
+                for (i = 0; i < cnt; i++) {
+                        subvol = layout->list[i].xlator;
+                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
+                                    subvol, subvol->fops->getxattr,
+                                    loc, key, NULL);
+                }
+                return 0;
+        }
+
+        /* node-uuid or pathinfo for files */
+        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
+                    || XATTR_IS_PATHINFO (key))) {
                 cached_subvol = local->cached_subvol;
                 (void) strncpy (local->xsel, key, 256);
 
                 local->call_cnt = 1;
                 STACK_WIND (frame, dht_vgetxattr_cbk, cached_subvol,
-                            cached_subvol->fops->getxattr, loc, key);
+                            cached_subvol->fops->getxattr, loc, key, NULL);
 
                 return 0;
         }
+
         if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
                 if (!hashed_subvol) {
@@ -1918,7 +2220,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 if (hashed_subvol) {
                         STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
                                     hashed_subvol->fops->getxattr, loc,
-                                    GF_XATTR_PATHINFO_KEY);
+                                    GF_XATTR_PATHINFO_KEY, NULL);
                         return 0;
                 }
                 op_errno = ENODATA;
@@ -1926,13 +2228,13 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         }
 
         if (key && (!strcmp (GF_XATTR_MARKER_KEY, key))
-            && (-1 == frame->root->pid)) {
-
-                if (loc->inode-> ia_type == IA_IFDIR) {
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                if (DHT_IS_DIR(layout)) {
                         cnt = layout->cnt;
                 } else {
                         cnt = 1;
                 }
+
                 sub_volumes = alloca ( cnt * sizeof (xlator_t *));
                 for (i = 0; i < cnt; i++)
                         *(sub_volumes + i) = layout->list[i].xlator;
@@ -1940,7 +2242,8 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 if (cluster_getmarkerattr (frame, this, loc, key,
                                            local, dht_getxattr_unwind,
                                            sub_volumes, cnt,
-                                           MARKER_UUID_TYPE, conf->vol_uuid)) {
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           conf->vol_uuid)) {
                         op_errno = EINVAL;
                         goto err;
                 }
@@ -1948,10 +2251,22 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
+        if (key && !strcmp (GF_XATTR_QUOTA_LIMIT_LIST, key)) {
+                /* quota hardlimit and aggregated size of a directory is stored
+                 * in inode contexts of each brick. Hence its good enough that
+                 * we send getxattr for this key to any brick.
+                 */
+                local->call_cnt = 1;
+                subvol = dht_first_up_subvol (this);
+                STACK_WIND (frame, dht_getxattr_cbk, subvol,
+                            subvol->fops->getxattr, loc, key, xdata);
+                return 0;
+        }
+
         if (key && *conf->vol_uuid) {
                 if ((match_uuid_local (key, conf->vol_uuid) == 0) &&
-                    (-1 == frame->root->pid)) {
-                        if (loc->inode-> ia_type == IA_IFDIR) {
+                    (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                        if (DHT_IS_DIR(layout)) {
                                 cnt = layout->cnt;
                         } else {
                                 cnt = 1;
@@ -1964,6 +2279,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                                                    local, dht_getxattr_unwind,
                                                    sub_volumes, cnt,
                                                    MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
                                                    conf->vol_uuid)) {
                                 op_errno = EINVAL;
                                 goto err;
@@ -1973,7 +2289,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (loc->inode-> ia_type == IA_IFDIR) {
+        if (DHT_IS_DIR(layout)) {
                 cnt = local->call_cnt = layout->cnt;
         } else {
                 cnt = local->call_cnt  = 1;
@@ -1983,20 +2299,21 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_getxattr_cbk,
                             subvol, subvol->fops->getxattr,
-                            loc, key);
+                            loc, key, NULL);
         }
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
+#undef DHT_IS_DIR
 
 int
 dht_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *key)
+               fd_t *fd, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol        = NULL;
         dht_local_t  *local         = NULL;
@@ -2034,7 +2351,10 @@ dht_fgetxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (fd->inode->ia_type == IA_IFDIR) {
+        if ((fd->inode->ia_type == IA_IFDIR)
+	    && key
+            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
+                         strlen (GF_XATTR_LOCKINFO_KEY) != 0))) {
                 cnt = local->call_cnt = layout->cnt;
         } else {
                 cnt = local->call_cnt  = 1;
@@ -2044,33 +2364,36 @@ dht_fgetxattr (call_frame_t *frame, xlator_t *this,
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_getxattr_cbk,
                             subvol, subvol->fops->fgetxattr,
-                            fd, key);
+                            fd, key, NULL);
         }
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *xattr, int flags)
+               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
 {
         xlator_t     *subvol   = NULL;
         dht_local_t  *local    = NULL;
         int           op_errno = EINVAL;
-        data_pair_t  *trav     = NULL;
+        dht_conf_t   *conf     = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
         VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
 
         local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
         if (!local) {
@@ -2089,13 +2412,13 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
-                    fd, xattr, flags);
+                    fd, xattr, flags, NULL);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2103,16 +2426,18 @@ err:
 
 static int
 dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno)
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *xdata)
 {
-        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
 int
 dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         int           i     = -1;
         int           ret   = -1;
@@ -2144,7 +2469,7 @@ dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP);
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
         }
         return 0;
 
@@ -2152,7 +2477,7 @@ out:
 
 int
 dht_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags)
+              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
 {
         xlator_t     *subvol   = NULL;
         dht_local_t  *local    = NULL;
@@ -2166,19 +2491,17 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         char          value[4096] = {0,};
         gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
         int           call_cnt = 0;
-        data_pair_t  *trav     = NULL;
-
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
 
         conf   = this->private;
+
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
+
         local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
         if (!local) {
                 op_errno = ENOMEM;
@@ -2203,25 +2526,6 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
 
         local->call_cnt = call_cnt = layout->cnt;
 
-        /*  This key is sent by Unified File and Object storage
-         *  to test xattr support in backend.
-         */
-        tmp = dict_get (xattr, "user.ufo-test");
-        if (tmp) {
-                if (IA_ISREG (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
-                local->op_ret = 0;
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_ufo_xattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setxattr,
-                                    loc, xattr, flags);
-                }
-                return 0;
-        }
-
         tmp = dict_get (xattr, "distribute.migrate-data");
         if (tmp) {
                 if (IA_ISDIR (loc->inode->ia_type)) {
@@ -2286,7 +2590,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, dht_checking_pathinfo_cbk,
                                     conf->subvolumes[i],
                                     conf->subvolumes[i]->fops->getxattr,
-                                    loc, GF_XATTR_PATHINFO_KEY);
+                                    loc, GF_XATTR_PATHINFO_KEY, NULL);
                 }
                 return 0;
         }
@@ -2296,9 +2600,13 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_INFO,
                         "fixing the layout of %s", loc->path);
 
-                dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
-                                          layout);
-                return 0;
+                ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
+                                                layout);
+                if (ret) {
+                        op_errno = ENOTCONN;
+                        goto err;
+                }
+                return ret;
         }
 
         tmp = dict_get (xattr, "distribute.directory-spread-count");
@@ -2310,10 +2618,14 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                              (dir_spread > 0))) {
                         layout->spread_cnt = dir_spread;
 
-                        dht_fix_directory_layout (frame,
-                                                  dht_common_setxattr_cbk,
-                                                  layout);
-                        return 0;
+                        ret = dht_fix_directory_layout (frame,
+                                                        dht_common_setxattr_cbk,
+                                                        layout);
+                        if (ret) {
+                                op_errno = ENOTCONN;
+                                goto err;
+                        }
+                        return ret;
                 }
                 gf_log (this->name, GF_LOG_ERROR,
                         "wrong 'directory-spread-count' value (%s)", value);
@@ -2325,14 +2637,14 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_err_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->setxattr,
-                            loc, xattr, flags);
+                            loc, xattr, flags, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2340,7 +2652,7 @@ err:
 
 int
 dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno)
+                     int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -2366,7 +2678,8 @@ unlock:
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
         }
 
         return 0;
@@ -2375,25 +2688,27 @@ unlock:
 
 int
 dht_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *key)
+                 loc_t *loc, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
+        dht_conf_t   *conf = NULL;
 
         int i;
 
         VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.dht*",
-                                 key, op_errno, err);
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
         if (!local) {
@@ -2424,34 +2739,37 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_removexattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->removexattr,
-                            loc, key);
+                            loc, key, NULL);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
 
 int
 dht_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *key)
+                  fd_t *fd, const char *key, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
+        dht_conf_t   *conf = 0;
 
         int i;
 
         VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.dht*",
-                                 key, op_errno, err);
+        conf = this->private;
+
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
         VALIDATE_OR_GOTO (frame, err);
 
@@ -2485,14 +2803,14 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, dht_removexattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->fremovexattr,
-                            fd, key);
+                            fd, key, NULL);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -2500,7 +2818,7 @@ err:
 
 int
 dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, fd_t *fd)
+            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -2527,7 +2845,7 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
                 DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                                  local->fd);
+                                  local->fd, NULL);
 
         return 0;
 }
@@ -2557,13 +2875,18 @@ dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
 
 int
 dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct statvfs *statvfs)
+                int op_ret, int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-        dht_local_t *local         = NULL;
-        int          this_call_cnt = 0;
-        int          bsize         = 0;
-        int          frsize        = 0;
+        dht_local_t *local              = NULL;
+        int          this_call_cnt      = 0;
+        int          bsize              = 0;
+        int          frsize             = 0;
+        int8_t       quota_deem_statfs  = 0;
+        GF_UNUSED int     ret           = 0;
+        unsigned long     new_usage     = 0;
+        unsigned long     cur_usage     = 0;
 
+        ret = dict_get_int8 (xdata, "quota-deem-statfs", &quota_deem_statfs);
 
         local = frame->local;
 
@@ -2573,8 +2896,22 @@ dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_errno = op_errno;
                         goto unlock;
                 }
+                if (!statvfs) {
+                        op_errno = EINVAL;
+                        local->op_ret = -1;
+                        goto unlock;
+                }
                 local->op_ret = 0;
 
+                if (quota_deem_statfs) {
+                        new_usage = statvfs->f_blocks - statvfs->f_bfree;
+                        cur_usage = local->statvfs.f_blocks - local->statvfs.f_bfree;
+                        /* We take the maximux of the usage from the subvols */
+                        if (new_usage >= cur_usage)
+                                local->statvfs = *statvfs;
+                        goto unlock;
+                }
+
                 if (local->statvfs.f_bsize != 0) {
                         bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
                         frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
@@ -2595,6 +2932,7 @@ dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->statvfs.f_flag     = statvfs->f_flag;
                 local->statvfs.f_namemax  = statvfs->f_namemax;
 
+
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -2603,14 +2941,14 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
                 DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->statvfs);
+                                  &local->statvfs, xdata);
 
         return 0;
 }
 
 
 int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         dht_local_t  *local  = NULL;
@@ -2622,7 +2960,6 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf = this->private;
@@ -2639,7 +2976,8 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 for (i = 0; i < conf->subvolume_cnt; i++) {
                         STACK_WIND (frame, dht_statfs_cbk,
                                     conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->statfs, loc);
+                                    conf->subvolumes[i]->fops->statfs, loc,
+                                    xdata);
                 }
                 return 0;
         }
@@ -2655,20 +2993,21 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_statfs_cbk,
-                    subvol, subvol->fops->statfs, loc);
+                    subvol, subvol->fops->statfs, loc, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -2695,14 +3034,14 @@ dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
                 STACK_WIND (frame, dht_fd_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->opendir,
-                            loc, fd);
+                            loc, fd, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -2710,7 +3049,7 @@ err:
 
 int
 dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries)
+                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         gf_dirent_t   entries;
@@ -2723,6 +3062,7 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
         dht_layout_t *layout = 0;
         dht_conf_t   *conf   = NULL;
         xlator_t     *subvol = 0;
+        int           ret    = 0;
 
         INIT_LIST_HEAD (&entries.list);
         prev = cookie;
@@ -2739,10 +3079,13 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
 
         list_for_each_entry (orig_entry, (&orig_entries->list), list) {
                 next_offset = orig_entry->d_off;
-                if ((check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
-                     (prev->this != dht_first_up_subvol (this))) ||
-                    check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict)) {
+                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
+                    (prev->this != local->first_up_subvol)) {
+                        continue;
+                }
+                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
+                                       orig_entry->dict,
+                                       conf->link_xattr_name)) {
                         continue;
                 }
 
@@ -2771,6 +3114,24 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 entry->d_type = orig_entry->d_type;
                 entry->d_len  = orig_entry->d_len;
 
+                if (orig_entry->dict)
+                        entry->dict = dict_ref (orig_entry->dict);
+
+                /* making sure we set the inode ctx right with layout,
+                   currently possible only for non-directories, so for
+                   directories don't set entry inodes */
+                if (!IA_ISDIR(entry->d_stat.ia_type) && orig_entry->inode) {
+                        ret = dht_layout_preset (this, prev->this,
+                                                 orig_entry->inode);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to link the layout in inode");
+                        entry->inode = inode_ref (orig_entry->inode);
+                } else if (orig_entry->inode) {
+                        dht_inode_ctx_time_update (orig_entry->inode, this,
+                                                   &entry->d_stat, 1);
+                }
+
                 list_add_tail (&entry->list, &entries.list);
                 count++;
         }
@@ -2800,6 +3161,19 @@ done:
                         goto unwind;
                 }
 
+		if (conf->readdir_optimize == _gf_true) {
+                        if (next_subvol != local->first_up_subvol) {
+                                ret = dict_set_int32 (local->xattr,
+                                                      GF_READDIR_SKIP_DIRS, 1);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+					         "dict set failed");
+		        } else {
+                                 dict_del (local->xattr,
+                                           GF_READDIR_SKIP_DIRS);
+                        }
+                }
+
                 STACK_WIND (frame, dht_readdirp_cbk,
                             next_subvol, next_subvol->fops->readdirp,
                             local->fd, local->size, next_offset,
@@ -2811,7 +3185,7 @@ unwind:
         if (op_ret < 0)
                 op_ret = 0;
 
-        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
 
         gf_dirent_free (&entries);
 
@@ -2822,7 +3196,8 @@ unwind:
 
 int
 dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries)
+                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
+                 dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         gf_dirent_t   entries;
@@ -2899,7 +3274,7 @@ done:
 
                 STACK_WIND (frame, dht_readdir_cbk,
                             next_subvol, next_subvol->fops->readdir,
-                            local->fd, local->size, next_offset);
+                            local->fd, local->size, next_offset, NULL);
                 return 0;
         }
 
@@ -2907,7 +3282,7 @@ unwind:
         if (op_ret < 0)
                 op_ret = 0;
 
-        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
 
         gf_dirent_free (&entries);
 
@@ -2924,10 +3299,14 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         xlator_t     *xvol = NULL;
         off_t         xoff = 0;
         int           ret = 0;
+        dht_conf_t   *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
         local = dht_local_init (frame, NULL, NULL, whichop);
         if (!local) {
@@ -2938,6 +3317,7 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         local->fd = fd_ref (fd);
         local->size = size;
         local->xattr_req = (dict)? dict_ref (dict) : NULL;
+        local->first_up_subvol = dht_first_up_subvol (this);
 
         dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
 
@@ -2950,26 +3330,38 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
                 if (local->xattr) {
                         ret = dict_set_uint32 (local->xattr,
-                                               "trusted.glusterfs.dht.linkto",
-                                               256);
+                                               conf->link_xattr_name, 256);
                         if (ret)
                                 gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to set 'glusterfs.dht.linkto'"
-                                        " key");
+                                        "failed to set '%s' key",
+                                        conf->link_xattr_name);
+			if (conf->readdir_optimize == _gf_true) {
+                                if (xvol != local->first_up_subvol) {
+				        ret = dict_set_int32 (local->xattr,
+			                               GF_READDIR_SKIP_DIRS, 1);
+				        if (ret)
+					        gf_log (this->name,
+                                                        GF_LOG_ERROR,
+						        "Dict set failed");
+                                } else {
+                                        dict_del (local->xattr,
+                                                  GF_READDIR_SKIP_DIRS);
+                                }
+			}
                 }
 
                 STACK_WIND (frame, dht_readdirp_cbk, xvol, xvol->fops->readdirp,
                             fd, size, xoff, local->xattr);
         } else {
                 STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, xoff);
+                            fd, size, xoff, local->xattr);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -2977,7 +3369,7 @@ err:
 
 int
 dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff)
+             off_t yoff, dict_t *xdata)
 {
         int          op = GF_FOP_READDIR;
         dht_conf_t  *conf = NULL;
@@ -3014,7 +3406,7 @@ dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
 int
 dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno)
+                  int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -3034,14 +3426,16 @@ dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret, local->op_errno);
+                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
 int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              int datasync, dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -3068,14 +3462,14 @@ dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
                 STACK_WIND (frame, dht_fsyncdir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->fsyncdir,
-                            fd, datasync);
+                            fd, datasync, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -3085,9 +3479,9 @@ int
 dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int op_ret, int op_errno,
                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
-        call_frame_t *prev = NULL;
+        xlator_t     *prev = NULL;
         int           ret = -1;
         dht_local_t  *local = NULL;
 
@@ -3105,19 +3499,24 @@ dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
-        ret = dht_layout_preset (this, prev->this, inode);
+        ret = dht_layout_preset (this, prev, inode);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "could not set pre-set layout for subvolume %s",
-                        prev->this->name);
+                        prev? prev->name: NULL);
                 op_ret   = -1;
                 op_errno = EINVAL;
                 goto out;
         }
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 out:
         /*
          * FIXME: ia_size and st_blocks of preparent and postparent do not have
@@ -3126,10 +3525,9 @@ out:
          * corresponding values from each of the subvolume.
          * See dht_iatt_merge for reference.
          */
-
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf,
+                          preparent, postparent, xdata);
         return 0;
 }
 
@@ -3138,7 +3536,8 @@ dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this,
                                int32_t op_ret, int32_t op_errno,
                                inode_t *inode, struct iatt *stbuf,
-                               struct iatt *preparent, struct iatt *postparent)
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -3147,22 +3546,28 @@ dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 goto err;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
         cached_subvol = local->cached_subvol;
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    cached_subvol, cached_subvol->fops->mknod,
-                    &local->loc, local->mode, local->rdev,
-                    local->params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
+                           cached_subvol, cached_subvol->fops->mknod,
+                           &local->loc, local->mode, local->rdev, local->umask,
+                           local->params);
 
         return 0;
 err:
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
 int
 dht_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
         xlator_t    *subvol = NULL;
         int          op_errno = -1;
@@ -3194,11 +3599,13 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s", loc->path, subvol->name);
 
-                STACK_WIND (frame, dht_newfile_cbk,
-                            subvol, subvol->fops->mknod,
-                            loc, mode, rdev, params);
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
+                                   subvol, subvol->fops->mknod, loc, mode,
+                                   rdev, umask, params);
         } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol);
+
+                avail_subvol = dht_free_disk_available_subvol (this, subvol,
+                                                               local);
                 if (avail_subvol != subvol) {
                         /* Choose the minimum filled volume, and create the
                            files there */
@@ -3207,17 +3614,18 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
                         local->cached_subvol = avail_subvol;
                         local->mode = mode;
                         local->rdev = rdev;
-
+                        local->umask = umask;
                         dht_linkfile_create (frame,
                                              dht_mknod_linkfile_create_cbk,
-                                             avail_subvol, subvol, loc);
+                                             this, avail_subvol, subvol, loc);
                 } else {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "creating %s on %s", loc->path, subvol->name);
 
-                        STACK_WIND (frame, dht_newfile_cbk,
-                                    subvol, subvol->fops->mknod,
-                                    loc, mode, rdev, params);
+                        STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                                           (void *)subvol, subvol,
+                                           subvol->fops->mknod, loc, mode,
+                                           rdev, umask, params);
                 }
         }
 
@@ -3226,7 +3634,7 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3234,7 +3642,7 @@ err:
 
 int
 dht_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkname, loc_t *loc, dict_t *params)
+             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
 {
         xlator_t    *subvol = NULL;
         int          op_errno = -1;
@@ -3262,23 +3670,24 @@ dht_symlink (call_frame_t *frame, xlator_t *this,
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->symlink,
-                    linkname, loc, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->symlink, linkname, loc, umask,
+                           params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (link, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+            dict_t *xdata)
 {
         xlator_t    *cached_subvol = NULL;
         xlator_t    *hashed_subvol = NULL;
@@ -3296,7 +3705,7 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                         local->loc.path, cached_subvol->name, loc->path);
                 STACK_WIND (frame, dht_unlink_cbk,
                             cached_subvol, cached_subvol->fops->unlink,
-                            &local->loc);
+                            &local->loc, xflag, xdata);
                 goto done;
         }
 
@@ -3308,12 +3717,12 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         hashed_subvol = dht_subvol_get_hashed (this, loc);
+        /* Dont fail unlink if hashed_subvol is NULL which can be the result
+         * of layout anomaly */
         if (!hashed_subvol) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "no subvolume in layout for path=%s",
                         loc->path);
-                op_errno = EINVAL;
-                goto err;
         }
 
         cached_subvol = local->cached_subvol;
@@ -3324,18 +3733,21 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto err;
         }
 
-        if (hashed_subvol != cached_subvol) {
+        local->flags = xflag;
+        if (hashed_subvol && hashed_subvol != cached_subvol) {
                 STACK_WIND (frame, dht_unlink_linkfile_cbk,
-                            hashed_subvol, hashed_subvol->fops->unlink, loc);
+                            hashed_subvol, hashed_subvol->fops->unlink, loc,
+                            xflag, xdata);
         } else {
                 STACK_WIND (frame, dht_unlink_cbk,
-                            cached_subvol, cached_subvol->fops->unlink, loc);
+                            cached_subvol, cached_subvol->fops->unlink, loc,
+                            xflag, xdata);
         }
 done:
         return 0;
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3345,13 +3757,16 @@ int
 dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int op_ret, int op_errno,
               inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent)
+              struct iatt *postparent, dict_t *xdata)
 {
         call_frame_t *prev = NULL;
         dht_layout_t *layout = NULL;
+        dht_local_t  *local = NULL;
 
         prev = cookie;
 
+        local = frame->local;
+
         if (op_ret == -1)
                 goto out;
 
@@ -3365,13 +3780,20 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        WIPE (preparent);
-        WIPE (postparent);
-
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
 
         return 0;
 }
@@ -3381,7 +3803,8 @@ int
 dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
                        inode_t *inode, struct iatt *stbuf,
-                       struct iatt *preparent, struct iatt *postparent)
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *srcvol = NULL;
@@ -3393,14 +3816,14 @@ dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         srcvol = local->linkfile.srcvol;
 
         STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
-                    &local->loc, &local->loc2);
+                    &local->loc, &local->loc2, xdata);
 
         return 0;
 
 err:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
 
         return 0;
 }
@@ -3408,7 +3831,7 @@ err:
 
 int
 dht_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc)
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         xlator_t    *cached_subvol = NULL;
         xlator_t    *hashed_subvol = NULL;
@@ -3453,19 +3876,19 @@ dht_link (call_frame_t *frame, xlator_t *this,
 
         if (hashed_subvol != cached_subvol) {
                 uuid_copy (local->gfid, oldloc->inode->gfid);
-                dht_linkfile_create (frame, dht_link_linkfile_cbk,
+                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
                                      cached_subvol, hashed_subvol, newloc);
         } else {
                 STACK_WIND (frame, dht_link_cbk,
                             cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -3475,7 +3898,7 @@ int
 dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno,
                 fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent)
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         call_frame_t *prev = NULL;
         int           ret = -1;
@@ -3494,8 +3917,11 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
         ret = dht_layout_preset (this, prev->this, inode);
@@ -3507,11 +3933,14 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 op_errno = EINVAL;
                 goto out;
         }
-
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
         return 0;
 }
 
@@ -3521,7 +3950,8 @@ dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this,
                                 int32_t op_ret, int32_t op_errno,
                                 inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent)
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -3535,18 +3965,19 @@ dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
         STACK_WIND (frame, dht_create_cbk,
                     cached_subvol, cached_subvol->fops->create,
                     &local->loc, local->flags, local->mode,
-                    local->fd, local->params);
+                    local->umask, local->fd, local->params);
 
         return 0;
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
         return 0;
 }
 
 int
 dht_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+            mode_t umask, fd_t *fd, dict_t *params)
 {
         int          op_errno = -1;
         xlator_t    *subvol = NULL;
@@ -3572,7 +4003,7 @@ dht_create (call_frame_t *frame, xlator_t *this,
                         local->loc.path, subvol->name, loc->path);
                 STACK_WIND (frame, dht_create_cbk,
                             subvol, subvol->fops->create,
-                            &local->loc, flags, mode, fd, params);
+                            &local->loc, flags, mode, umask, fd, params);
                 goto done;
         }
 
@@ -3590,38 +4021,38 @@ dht_create (call_frame_t *frame, xlator_t *this,
                         "creating %s on %s", loc->path, subvol->name);
                 STACK_WIND (frame, dht_create_cbk,
                             subvol, subvol->fops->create,
-                            loc, flags, mode, fd, params);
+                            loc, flags, mode, umask, fd, params);
                 goto done;
         }
         /* Choose the minimum filled volume, and create the
            files there */
-        avail_subvol = dht_free_disk_available_subvol (this, subvol);
+        avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
         if (avail_subvol != subvol) {
                 local->params = dict_ref (params);
                 local->flags = flags;
                 local->mode = mode;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
                 local->hashed_subvol = subvol;
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s (link at %s)", loc->path,
                         avail_subvol->name, subvol->name);
-                dht_linkfile_create (frame,
-                                     dht_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 goto done;
         }
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 done:
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
 
         return 0;
 }
@@ -3630,7 +4061,7 @@ err:
 int
 dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
                         xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         dht_layout_t  *layout = NULL;
@@ -3641,14 +4072,17 @@ dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
         if (op_ret == 0) {
                 dht_layout_set (this, local->inode, layout);
                 if (local->loc.parent) {
-                        WIPE (&local->preparent);
-                        WIPE (&local->postparent);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
                 }
         }
 
         DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
                           local->inode, &local->stbuf, &local->preparent,
-                          &local->postparent);
+                          &local->postparent, NULL);
 
         return 0;
 }
@@ -3656,7 +4090,7 @@ dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
 int
 dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-               struct iatt *preparent, struct iatt *postparent)
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -3677,6 +4111,15 @@ dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 -1, ENOSPC, NULL);
                 } else {
+			if (op_ret == -1 && op_errno == EEXIST)
+				/* Very likely just a race between mkdir and
+				   self-heal (from lookup of a concurrent mkdir
+				   attempt).
+				   Ignore error for now. layout setting will
+				   anyways fail if this was a different (old)
+				   pre-existing different directory.
+				*/
+				op_ret = 0;
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 op_ret, op_errno, NULL);
                 }
@@ -3709,7 +4152,8 @@ int
 dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
                       xlator_t *this, int op_ret, int op_errno,
                       inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           ret = -1;
@@ -3766,19 +4210,20 @@ dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
                         continue;
                 STACK_WIND (frame, dht_mkdir_cbk,
                             conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->mkdir,
-                            &local->loc, local->mode, local->params);
+                            conf->subvolumes[i]->fops->mkdir, &local->loc,
+                            local->mode, local->umask, local->params);
         }
         return 0;
 err:
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
         return 0;
 }
 
 
-int
+ int
 dht_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dict_t *params)
+           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -3814,6 +4259,7 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
 
         local->hashed_subvol = hashed_subvol;
         local->mode = mode;
+        local->umask = umask;
         local->params = dict_ref (params);
         local->inode  = inode_ref (loc->inode);
 
@@ -3826,13 +4272,14 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, dht_mkdir_hashed_cbk,
                     hashed_subvol,
                     hashed_subvol->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
 
         return 0;
 }
@@ -3840,14 +4287,87 @@ err:
 
 int
 dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno)
+                        int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
         local = frame->local;
 
         DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
+
+        return 0;
+}
+
+
+int
+dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret   = -1;
+                        if (op_errno != ENOENT && op_errno != EACCES) {
+                                local->need_selfheal = 1;
+                        }
+
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "rmdir on %s for %s failed (%s)",
+                                prev->this->name, local->loc.path,
+                                strerror (op_errno));
+                        goto unlock;
+                }
+
+                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
+
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+               if (local->need_selfheal) {
+                        local->layout =
+                                dht_layout_get (this, local->loc.inode);
+
+                        /* TODO: neater interface needed below */
+                        local->stbuf.ia_type = local->loc.inode->ia_type;
+
+                        uuid_copy (local->gfid, local->loc.inode->gfid);
+                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
+                                              &local->loc, local->layout);
+               } else {
+
+                        if (local->loc.parent) {
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+                        }
+
+                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
+                                          local->op_errno, &local->preparent,
+                                          &local->postparent, NULL);
+               }
+        }
 
         return 0;
 }
@@ -3856,11 +4376,12 @@ dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
         call_frame_t *prev = NULL;
+        int           done = 0;
 
         local = frame->local;
         prev  = cookie;
@@ -3893,7 +4414,16 @@ unlock:
 
 
         this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
+
+        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+        if (local->hashed_subvol && (this_call_cnt == 1)) {
+                done = 1;
+        } else if (!local->hashed_subvol && !this_call_cnt) {
+                done = 1;
+        }
+
+
+        if (done) {
                 if (local->need_selfheal && local->fop_succeeded) {
                         local->layout =
                                 dht_layout_get (this, local->loc.inode);
@@ -3904,15 +4434,34 @@ unlock:
                         uuid_copy (local->gfid, local->loc.inode->gfid);
                         dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
                                               &local->loc, local->layout);
-                } else {
+                } else if (this_call_cnt) {
+                        /* If non-hashed subvol's have responded, proceed */
+
+                        local->need_selfheal = 0;
+                        STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                                    local->hashed_subvol,
+                                    local->hashed_subvol->fops->rmdir,
+                                    &local->loc, local->flags, NULL);
+                } else if (!this_call_cnt) {
+                        /* All subvol's have responded, proceed */
+
                         if (local->loc.parent) {
-                                WIPE (&local->preparent);
-                                WIPE (&local->postparent);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+
                         }
 
                         DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
                                           local->op_errno, &local->preparent,
-                                          &local->postparent);
+                                          &local->postparent, NULL);
                 }
         }
 
@@ -3926,6 +4475,7 @@ dht_rmdir_do (call_frame_t *frame, xlator_t *this)
         dht_local_t  *local = NULL;
         dht_conf_t   *conf = NULL;
         int           i = 0;
+        xlator_t     *hashed_subvol = NULL;
 
         VALIDATE_OR_GOTO (this->private, err);
 
@@ -3937,18 +4487,41 @@ dht_rmdir_do (call_frame_t *frame, xlator_t *this)
 
         local->call_cnt = conf->subvolume_cnt;
 
+        /* first remove from non-hashed_subvol */
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to get hashed "
+                        "subvol for %s",local->loc.path);
+        } else {
+                local->hashed_subvol = hashed_subvol;
+        }
+
+        /* When DHT has only 1 child */
+        if (conf->subvolume_cnt == 1) {
+                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                            conf->subvolumes[0],
+                            conf->subvolumes[0]->fops->rmdir,
+                            &local->loc, local->flags, NULL);
+                return 0;
+        }
+
         for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (hashed_subvol &&
+                    (hashed_subvol == conf->subvolumes[i]))
+                        continue;
+
                 STACK_WIND (frame, dht_rmdir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->rmdir,
-                            &local->loc, local->flags);
+                            &local->loc, local->flags, NULL);
         }
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
         return 0;
 }
 
@@ -3956,7 +4529,7 @@ err:
 int
 dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                int op_ret, int op_errno, struct iatt *preparent,
-                               struct iatt *postparent)
+                               struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t    *local = NULL;
         call_frame_t   *prev = NULL;
@@ -4004,6 +4577,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_frame_t   *main_frame = NULL;
         dht_local_t    *main_local = NULL;
         int             this_call_cnt = 0;
+        dht_conf_t     *conf = this->private;
 
         local = frame->local;
         prev  = cookie;
@@ -4015,7 +4589,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret != 0)
                 goto err;
 
-        if (check_is_linkfile (inode, stbuf, xattr) == 0) {
+        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 main_local->op_ret  = -1;
                 main_local->op_errno = ENOTEMPTY;
 
@@ -4026,7 +4600,74 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_WIND (frame, dht_rmdir_linkfile_unlink_cbk,
-                    src, src->fops->unlink, &local->loc);
+                    src, src->fops->unlink, &local->loc, 0, NULL);
+        return 0;
+err:
+
+        this_call_cnt = dht_frame_return (main_frame);
+        if (is_last_call (this_call_cnt))
+                dht_rmdir_do (main_frame, this);
+
+        DHT_STACK_DESTROY (frame);
+        return 0;
+}
+
+
+int
+dht_rmdir_cached_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, inode_t *inode,
+                             struct iatt *stbuf, dict_t *xattr,
+                             struct iatt *parent)
+{
+        dht_local_t    *local         = NULL;
+        xlator_t       *src           = NULL;
+        call_frame_t   *main_frame    = NULL;
+        dht_local_t    *main_local    = NULL;
+        int             this_call_cnt = 0;
+        dht_conf_t     *conf          = this->private;
+        dict_t         *xattrs        = NULL;
+        int             ret           = 0;
+
+        local = frame->local;
+        src   = local->hashed_subvol;
+
+        main_frame = local->main_frame;
+        main_local = main_frame->local;
+
+        if (op_ret == 0) {
+                main_local->op_ret  = -1;
+                main_local->op_errno = ENOTEMPTY;
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s found on cached subvol %s",
+                        local->loc.path, src->name);
+                goto err;
+        } else if (op_errno != ENOENT) {
+                main_local->op_ret  = -1;
+                main_local->op_errno = op_errno;
+                goto err;
+        }
+
+        xattrs = dict_new ();
+        if (!xattrs) {
+                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+                goto err;
+        }
+
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
+                        " in dict");
+                if (xattrs)
+                        dict_unref (xattrs);
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_rmdir_lookup_cbk,
+                    src, src->fops->lookup, &local->loc, xattrs);
+        if (xattrs)
+                dict_unref (xattrs);
+
         return 0;
 err:
 
@@ -4043,12 +4684,15 @@ int
 dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                            gf_dirent_t *entries, xlator_t *src)
 {
-        int                 ret = 0;
-        int                 build_ret = 0;
-        gf_dirent_t        *trav = NULL;
+        int                 ret          = 0;
+        int                 build_ret    = 0;
+        gf_dirent_t        *trav         = NULL;
         call_frame_t       *lookup_frame = NULL;
         dht_local_t        *lookup_local = NULL;
-        dht_local_t        *local = NULL;
+        dht_local_t        *local        = NULL;
+        dict_t             *xattrs       = NULL;
+        dht_conf_t         *conf         = this->private;
+        xlator_t           *subvol       = NULL;
 
         local = frame->local;
 
@@ -4057,7 +4701,8 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                         continue;
                 if (strcmp (trav->d_name, "..") == 0)
                         continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict)) {
+                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
+                                              conf->link_xattr_name)) {
                         ret++;
                         continue;
                 }
@@ -4069,6 +4714,21 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
+        xattrs = dict_new ();
+        if (!xattrs) {
+                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+                return -1;
+        }
+
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
+                        " in dict");
+                if (xattrs)
+                        dict_unref (xattrs);
+                return -1;
+        }
+
         list_for_each_entry (trav, &entries->list, list) {
                 if (strcmp (trav->d_name, ".") == 0)
                         continue;
@@ -4092,6 +4752,7 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
 
                 lookup_frame->local = lookup_local;
                 lookup_local->main_frame = frame;
+                lookup_local->hashed_subvol = src;
 
                 build_ret = dht_build_child_loc (this, &lookup_local->loc,
                                                  &local->loc, trav->d_name);
@@ -4110,14 +4771,31 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                 }
                 UNLOCK (&frame->lock);
 
-                STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
-                            src, src->fops->lookup,
-                            &lookup_local->loc, NULL);
+                subvol = dht_linkfile_subvol (this, NULL, &trav->d_stat,
+                                              trav->dict);
+                if (!subvol) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "linkfile not having link subvolume. path=%s",
+                                lookup_local->loc.path);
+                        STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
+                                    src, src->fops->lookup,
+                                    &lookup_local->loc, xattrs);
+                } else {
+                        STACK_WIND (lookup_frame, dht_rmdir_cached_lookup_cbk,
+                                    subvol, subvol->fops->lookup,
+                                    &lookup_local->loc, xattrs);
+                }
                 ret++;
         }
 
+        if (xattrs)
+                dict_unref (xattrs);
+
         return ret;
 err:
+        if (xattrs)
+                dict_unref (xattrs);
+
         DHT_STACK_DESTROY (lookup_frame);
         return 0;
 }
@@ -4125,7 +4803,8 @@ err:
 
 int
 dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries)
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
@@ -4169,27 +4848,38 @@ dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, fd_t *fd)
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+        dht_local_t  *local         = NULL;
         int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-        dict_t       *dict = NULL;
-        int           ret = 0;
+        call_frame_t *prev          = NULL;
+        dict_t       *dict          = NULL;
+        int           ret           = 0;
+        dht_conf_t   *conf          = this->private;
+        int           i             = 0;
 
         local = frame->local;
         prev  = cookie;
 
+        this_call_cnt = dht_frame_return (frame);
         if (op_ret == -1) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "opendir on %s for %s failed (%s)",
                         prev->this->name, local->loc.path,
                         strerror (op_errno));
-                local->op_ret = -1;
-                local->op_errno = op_errno;
+                if (op_errno != ENOENT) {
+                        local->op_ret = -1;
+                        local->op_errno = op_errno;
+                }
                 goto err;
         }
 
+        if (!is_last_call (this_call_cnt))
+                return 0;
+
+        if (local->op_ret == -1)
+                goto err;
+
         dict = dict_new ();
         if (!dict) {
                 local->op_ret = -1;
@@ -4197,16 +4887,19 @@ dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto err;
         }
 
-        ret = dict_set_uint32 (dict,
-                               "trusted.glusterfs.dht.linkto", 256);
+        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        local->loc.path);
+                        "%s: failed to set '%s' key",
+                        local->loc.path, conf->link_xattr_name);
 
-        STACK_WIND (frame, dht_rmdir_readdirp_cbk,
-                    prev->this, prev->this->fops->readdirp,
-                    local->fd, 4096, 0, dict);
+        local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_rmdir_readdirp_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->readdirp,
+                            local->fd, 4096, 0, dict);
+        }
 
         if (dict)
                 dict_unref (dict);
@@ -4214,8 +4907,6 @@ dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 
 err:
-        this_call_cnt = dht_frame_return (frame);
-
         if (is_last_call (this_call_cnt)) {
                 dht_rmdir_do (frame, this);
         }
@@ -4225,7 +4916,8 @@ err:
 
 
 int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           dict_t *xdata)
 {
         dht_local_t  *local  = NULL;
         dht_conf_t   *conf = NULL;
@@ -4264,7 +4956,7 @@ dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
                 STACK_WIND (frame, dht_rmdir_opendir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->opendir,
-                            loc, local->fd);
+                            loc, local->fd, NULL);
         }
 
         return 0;
@@ -4272,17 +4964,17 @@ dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
-                          NULL, NULL);
+                          NULL, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -4290,7 +4982,7 @@ dht_entrylk_cbk (call_frame_t *frame, void *cookie,
 int
 dht_entrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type)
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -4300,7 +4992,6 @@ dht_entrylk (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
         if (!local) {
@@ -4320,13 +5011,13 @@ dht_entrylk (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_entrylk_cbk,
                     subvol, subvol->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -4334,10 +5025,10 @@ err:
 
 int
 dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
@@ -4345,7 +5036,7 @@ dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
 int
 dht_fentrylk (call_frame_t *frame, xlator_t *this,
               const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type)
+              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -4364,13 +5055,13 @@ dht_fentrylk (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_fentrylk_cbk,
                     subvol, subvol->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
+                    volume, fd, basename, cmd, type, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -4379,16 +5070,21 @@ err:
 int
 dht_forget (xlator_t *this, inode_t *inode)
 {
-        uint64_t      tmp_layout = 0;
+        uint64_t        ctx_int = 0;
+        dht_inode_ctx_t *ctx    = NULL;
         dht_layout_t *layout = NULL;
 
-        inode_ctx_del (inode, this, &tmp_layout);
+        inode_ctx_del (inode, this, &ctx_int);
 
-        if (!tmp_layout)
+        if (!ctx_int)
                 return 0;
 
-        layout = (dht_layout_t *)(long)tmp_layout;
+        ctx = (dht_inode_ctx_t *) (long) ctx_int;
+
+        layout = ctx->layout;
+        ctx->layout = NULL;
         dht_layout_unref (this, layout);
+        GF_FREE (ctx);
 
         return 0;
 }
@@ -4474,7 +5170,12 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                 if (conf->assert_no_child_down) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Received CHILD_DOWN. Exiting");
-                        kill (getpid(), SIGTERM);
+                        if (conf->defrag) {
+                                gf_defrag_stop (conf->defrag,
+                                                GF_DEFRAG_STATUS_FAILED, NULL);
+                        } else {
+                                kill (getpid(), SIGTERM);
+                        }
                 }
 
                 for (i = 0; i < conf->subvolume_cnt; i++) {
@@ -4547,7 +5248,8 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                         if (cmd == GF_DEFRAG_CMD_STATUS)
                                 gf_defrag_status_get (defrag, output);
                         else if (cmd == GF_DEFRAG_CMD_STOP)
-                                gf_defrag_stop (defrag, output);
+                                gf_defrag_stop (defrag,
+                                                GF_DEFRAG_STATUS_STOPPED, output);
                 }
 unlock:
                 UNLOCK (&defrag->lock);
@@ -4572,15 +5274,7 @@ unlock:
            or wait for anything else. Just propagate blindly */
         if (have_heard_from_all) {
                 propagate = 1;
-                if (conf->defrag) {
-                        ret = pthread_create (&conf->defrag->th, NULL,
-                                              gf_defrag_start, this);
-                        if (ret) {
-                                conf->defrag = NULL;
-                                GF_FREE (conf->defrag);
-                                kill (getpid(), SIGTERM);
-                        }
-                }
+
         }
 
 
@@ -4602,6 +5296,19 @@ unlock:
                                 /* continue to check other events for CHILD_UP */
                         }
                 }
+
+                /* rebalance is started with assert_no_child_down. So we do
+                 * not need to handle CHILD_DOWN event here.
+                 */
+                if (conf->defrag) {
+                        ret = gf_thread_create (&conf->defrag->th, NULL,
+						gf_defrag_start, this);
+                        if (ret) {
+                                conf->defrag = NULL;
+                                GF_FREE (conf->defrag);
+                                kill (getpid(), SIGTERM);
+                        }
+                }
         }
 
         ret = 0;
@@ -4610,3 +5317,24 @@ unlock:
 
         return ret;
 }
+
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (!ret && ctx) {
+                if (ctx->layout) {
+                        if (layout)
+                                *layout = ctx->layout;
+                        ret = 0;
+                } else {
+                        ret = -1;
+                }
+        }
+
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index d7689cc7f..d391b87d5 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,6 +13,9 @@
 #include "config.h"
 #endif
 
+#include <regex.h>
+#include <signal.h>
+
 #include "dht-mem-types.h"
 #include "libxlator.h"
 #include "syncop.h"
@@ -38,7 +32,8 @@
 
 typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
                                        xlator_t     *this,
-                                       int32_t       op_ret, int32_t op_errno);
+                                       int32_t       op_ret, int32_t op_errno,
+                                       dict_t *xdata);
 typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, call_frame_t *frame,
                                     int              ret);
 
@@ -61,20 +56,38 @@ struct dht_layout {
                 uint32_t   start;
                 uint32_t   stop;
                 xlator_t  *xlator;
-        } list[0];
+        } list[];
 };
 typedef struct dht_layout  dht_layout_t;
 
+struct dht_stat_time {
+        uint32_t        atime;
+        uint32_t        atime_nsec;
+        uint32_t        ctime;
+        uint32_t        ctime_nsec;
+        uint32_t        mtime;
+        uint32_t        mtime_nsec;
+};
+
+typedef struct dht_stat_time dht_stat_time_t;
+
+struct dht_inode_ctx {
+        dht_layout_t    *layout;
+        dht_stat_time_t  time;
+};
+
+typedef struct dht_inode_ctx dht_inode_ctx_t;
+
 
 typedef enum {
         DHT_HASH_TYPE_DM,
+        DHT_HASH_TYPE_DM_USER,
 } dht_hashfn_type_t;
 
 /* rebalance related */
 struct dht_rebalance_ {
         xlator_t            *from_subvol;
         xlator_t            *target_node;
-        int32_t              wbflags;
         off_t                offset;
         size_t               size;
         int32_t              flags;
@@ -83,6 +96,7 @@ struct dht_rebalance_ {
         struct iovec        *vector;
         struct iatt          stbuf;
         dht_defrag_cbk_fn_t  target_op_fn;
+        dict_t              *xdata;
 };
 
 struct dht_local {
@@ -129,7 +143,6 @@ struct dht_local {
         struct {
                 uint32_t         hole_cnt;
                 uint32_t         overlaps_cnt;
-                uint32_t         missing;
                 uint32_t         down;
                 uint32_t         misc;
                 dht_selfheal_dir_cbk_t   dir_cbk;
@@ -142,6 +155,7 @@ struct dht_local {
         int32_t flags;
         mode_t  mode;
         dev_t   rdev;
+        mode_t  umask;
 
         /* need for file-info */
         char   *xattr_val;
@@ -149,6 +163,7 @@ struct dht_local {
 
         /* which xattr request? */
         char xsel[256];
+        int32_t alloc_len;
 
         char   *newpath;
 
@@ -165,7 +180,12 @@ struct dht_local {
 
         glusterfs_fop_t      fop;
 
+        gf_boolean_t     linked;
+        xlator_t        *link_subvol;
+
         struct dht_rebalance_ rebalance;
+        xlator_t        *first_up_subvol;
+
 };
 typedef struct dht_local dht_local_t;
 
@@ -193,14 +213,27 @@ enum gf_defrag_status_t {
         GF_DEFRAG_STATUS_STOPPED,
         GF_DEFRAG_STATUS_COMPLETE,
         GF_DEFRAG_STATUS_FAILED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
+typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
+
+struct gf_defrag_pattern_list {
+        char                       path_pattern[256];
+        uint64_t                   size;
+        gf_defrag_pattern_list_t  *next;
+};
 
 struct gf_defrag_info_ {
         uint64_t                     total_files;
         uint64_t                     total_data;
         uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
+        uint64_t                     skipped;
         gf_lock_t                    lock;
         int                          cmd;
         pthread_t                    th;
@@ -211,7 +244,9 @@ struct gf_defrag_info_ {
         pid_t                        pid;
         inode_t                     *root_inode;
         uuid_t                       node_uuid;
-
+        struct timeval               start_time;
+        gf_boolean_t                 stats;
+        gf_defrag_pattern_list_t    *defrag_pattern;
 };
 
 typedef struct gf_defrag_info_ gf_defrag_info_t;
@@ -227,8 +262,8 @@ struct dht_conf {
         gf_boolean_t   search_unhashed;
         int            gen;
         dht_du_t      *du_stats;
-        uint64_t       min_free_disk;
-	uint32_t       min_free_inodes;
+        double         min_free_disk;
+	double         min_free_inodes;
         char           disk_unit;
         int32_t        refresh_interval;
         gf_boolean_t   unhashed_sticky_bit;
@@ -245,14 +280,28 @@ struct dht_conf {
         /* Will be a global flag to control the layout spread count */
         uint32_t       dir_spread_cnt;
 
-	struct syncenv *env; /* The env pointer to the rebalance synctask */
-
         /* to keep track of nodes which are decomissioned */
         xlator_t     **decommissioned_bricks;
         int            decommission_in_progress;
+        int            decommission_subvols_cnt;
 
         /* defrag related */
         gf_defrag_info_t *defrag;
+
+        /* Request to filter directory entries in readdir request */
+
+        gf_boolean_t    readdir_optimize;
+
+        /* Support regex-based name reinterpretation. */
+        regex_t         rsync_regex;
+        gf_boolean_t    rsync_regex_valid;
+        regex_t         extra_regex;
+        gf_boolean_t    extra_regex_valid;
+
+        /* Support variable xattr names. */
+        char            *xattr_name;
+        char            *link_xattr_name;
+        char            *wild_xattr_name;
 };
 typedef struct dht_conf dht_conf_t;
 
@@ -276,20 +325,14 @@ typedef enum {
 
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_revalidate(loc) (inode_ctx_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc) (dht_inode_ctx_layout_get (loc->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
 #define DHT_MIGRATION_IN_PROGRESS 1
 #define DHT_MIGRATION_COMPLETED   2
 
-#define DHT_LINKFILE_KEY         "trusted.glusterfs.dht.linkto"
-#define DHT_LINKFILE_MODE        (S_ISVTX)
-
-#define check_is_linkfile(i,s,x) (                                      \
-                ((st_mode_from_ia ((s)->ia_prot, (s)->ia_type) & ~S_IFMT) \
-                 == DHT_LINKFILE_MODE) &&                               \
-                dict_get (x, DHT_LINKFILE_KEY))
+#define check_is_linkfile(i,s,x,n) (IS_DHT_LINKFILE_MODE (s) && dict_get (x, n))
 
 #define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
                 IA_ISREG ((buf)->ia_type) &&                            \
@@ -308,6 +351,8 @@ typedef enum {
                 }                                               \
         } while (0)
 
+#define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE)
+
 #define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
@@ -334,6 +379,25 @@ typedef enum {
                 dht_local_wipe (__xl, __local);         \
         } while (0)
 
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
+                int32_t sec = 0;                                        \
+                sec = new_sec;                                          \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        new_sec = max(new_sec, ctx_sec);                \
+                        if (sec < new_sec)                              \
+                                new_nsec = ctx_nsec;                    \
+                        if (sec == new_sec)                             \
+                                new_nsec = max (new_nsec, ctx_nsec);    \
+                        if (post) {                                     \
+                                ctx_sec = new_sec;                      \
+                                ctx_nsec = new_nsec;                    \
+                        }                                               \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
 dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
 dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
 dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
@@ -343,7 +407,7 @@ int                                      dht_layout_normalize (xlator_t *this, l
 int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
                           uint32_t      *holes_p, uint32_t *overlaps_p,
                           uint32_t      *missing_p, uint32_t *down_p,
-                          uint32_t      *misc_p);
+                          uint32_t      *misc_p, uint32_t *no_space_p);
 int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
                              xlator_t   *subvol, loc_t *loc, dict_t *xattr);
 
@@ -377,12 +441,14 @@ int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt
 xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
 xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
 xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
+xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
 int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
 
-int dht_hash_compute (int type, const char *name, uint32_t *hash_p);
+int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
 int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
-                         xlator_t        *tovol, xlator_t *fromvol, loc_t *loc);
+                         xlator_t        *this, xlator_t *tovol,
+                         xlator_t        *fromvol, loc_t *loc);
 int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
@@ -397,17 +463,15 @@ dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
 int
 dht_layout_sort_volname (dht_layout_t *layout);
 
-int dht_rename (call_frame_t *frame, xlator_t *this,
-                loc_t *oldloc, loc_t *newloc);
-
 int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
 
 gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                          dht_local_t *layout);
 int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
 
 int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
-int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);
+int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
 void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
 dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
 xlator_t     *dht_first_up_subvol (xlator_t *this);
@@ -422,7 +486,8 @@ int                                     dht_rename_cleanup (call_frame_t *frame)
 int dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t           op_ret, int32_t op_errno,
                           inode_t          *inode, struct iatt *stbuf,
-                          struct iatt      *preparent, struct iatt *postparent);
+                          struct iatt      *preparent, struct iatt *postparent,
+                          dict_t *xdata);
 
 int dht_fix_directory_layout (call_frame_t *frame,
                               dht_selfheal_dir_cbk_t  dir_cbk,
@@ -445,73 +510,73 @@ int32_t dht_lookup (call_frame_t *frame,
 
 int32_t dht_stat (call_frame_t *frame,
                   xlator_t *this,
-                  loc_t    *loc);
+                  loc_t    *loc, dict_t *xdata);
 
 int32_t dht_fstat (call_frame_t *frame,
                    xlator_t *this,
-                   fd_t     *fd);
+                   fd_t     *fd, dict_t *xdata);
 
 int32_t dht_truncate (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
-                      off_t     offset);
+                      off_t     offset, dict_t *xdata);
 
 int32_t dht_ftruncate (call_frame_t *frame,
                        xlator_t *this,
                        fd_t     *fd,
-                       off_t     offset);
+                       off_t     offset, dict_t *xdata);
 
 int32_t dht_access (call_frame_t *frame,
                     xlator_t *this,
                     loc_t    *loc,
-                    int32_t   mask);
+                    int32_t   mask, dict_t *xdata);
 
 int32_t dht_readlink (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
-                      size_t    size);
+                      size_t    size, dict_t *xdata);
 
-int32_t dht_mknod (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, mode_t mode, dev_t rdev, dict_t *params);
+int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
 
 int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, mode_t mode, dict_t *params);
+                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t dht_unlink (call_frame_t *frame,
                     xlator_t *this,
-                    loc_t    *loc);
+                    loc_t    *loc, int xflag, dict_t *xdata);
 
 int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, int flags);
+                   loc_t    *loc, int flags, dict_t *xdata);
 
 int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
-                     const char *linkpath, loc_t *loc, dict_t *params);
+                     const char *linkpath, loc_t *loc, mode_t umask,
+                     dict_t *xdata);
 
 int32_t dht_rename (call_frame_t *frame,
                     xlator_t *this,
                     loc_t    *oldloc,
-                    loc_t    *newloc);
+                    loc_t    *newloc, dict_t *xdata);
 
 int32_t dht_link (call_frame_t *frame,
                   xlator_t *this,
                   loc_t    *oldloc,
-                  loc_t    *newloc);
+                  loc_t    *newloc, dict_t *xdata);
 
 int32_t dht_create (call_frame_t *frame, xlator_t *this,
                     loc_t    *loc, int32_t flags, mode_t mode,
-                    fd_t     *fd, dict_t *params);
+                    mode_t umask, fd_t     *fd, dict_t *params);
 
 int32_t dht_open (call_frame_t *frame,
                   xlator_t *this,
                   loc_t    *loc,
-                  int32_t   flags, fd_t *fd,
-                  int32_t   wbflags);
+                  int32_t   flags, fd_t *fd, dict_t *xdata);
 
 int32_t dht_readv (call_frame_t *frame,
                    xlator_t *this,
                    fd_t     *fd,
                    size_t    size,
-                   off_t     offset, uint32_t flags);
+                   off_t     offset, uint32_t flags, dict_t *xdata);
 
 int32_t dht_writev (call_frame_t      *frame,
                     xlator_t      *this,
@@ -520,87 +585,87 @@ int32_t dht_writev (call_frame_t      *frame,
                     int32_t        count,
                     off_t          offset,
                     uint32_t       flags,
-                    struct iobref *iobref);
+                    struct iobref *iobref, dict_t *xdata);
 
 int32_t dht_flush (call_frame_t *frame,
                    xlator_t *this,
-                   fd_t     *fd);
+                   fd_t     *fd, dict_t *xdata);
 
 int32_t dht_fsync (call_frame_t *frame,
                    xlator_t *this,
                    fd_t     *fd,
-                   int32_t   datasync);
+                   int32_t   datasync, dict_t *xdata);
 
 int32_t dht_opendir (call_frame_t *frame,
                      xlator_t *this,
-                     loc_t    *loc, fd_t *fd);
+                     loc_t    *loc, fd_t *fd, dict_t *xdata);
 
 int32_t dht_fsyncdir (call_frame_t *frame,
                       xlator_t *this,
                       fd_t     *fd,
-                      int32_t   datasync);
+                      int32_t   datasync, dict_t *xdata);
 
 int32_t dht_statfs (call_frame_t *frame,
                     xlator_t *this,
-                    loc_t    *loc);
+                    loc_t    *loc, dict_t *xdata);
 
 int32_t dht_setxattr (call_frame_t *frame,
                       xlator_t *this,
                       loc_t    *loc,
                       dict_t   *dict,
-                      int32_t   flags);
+                      int32_t   flags, dict_t *xdata);
 
 int32_t dht_getxattr (call_frame_t   *frame,
                       xlator_t   *this,
                       loc_t      *loc,
-                      const char *name);
+                      const char *name, dict_t *xdata);
 
 int32_t dht_fsetxattr (call_frame_t *frame,
                        xlator_t *this,
                        fd_t     *fd,
                        dict_t   *dict,
-                       int32_t   flags);
+                       int32_t   flags, dict_t *xdata);
 
 int32_t dht_fgetxattr (call_frame_t   *frame,
                        xlator_t   *this,
                        fd_t       *fd,
-                       const char *name);
+                       const char *name, dict_t *xdata);
 
 int32_t dht_removexattr (call_frame_t   *frame,
                          xlator_t   *this,
                          loc_t      *loc,
-                         const char *name);
+                         const char *name, dict_t *xdata);
 int32_t dht_fremovexattr (call_frame_t   *frame,
                           xlator_t   *this,
                           fd_t      *fd,
-                          const char *name);
+                          const char *name, dict_t *xdata);
 
 int32_t dht_lk (call_frame_t        *frame,
                 xlator_t        *this,
                 fd_t            *fd,
                 int32_t          cmd,
-                struct gf_flock *flock);
+                struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
                      const char      *volume, loc_t *loc, int32_t cmd,
-                     struct gf_flock *flock);
+                     struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
                       const char      *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *flock);
+                      struct gf_flock *flock, dict_t *xdata);
 
 int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
                      const char  *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd  cmd, entrylk_type type);
+                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
 
 int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
                       const char  *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd  cmd, entrylk_type type);
+                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
 
 int32_t dht_readdir (call_frame_t  *frame,
                      xlator_t *this,
                      fd_t     *fd,
-                     size_t    size, off_t off);
+                     size_t    size, off_t off, dict_t *xdata);
 
 int32_t dht_readdirp (call_frame_t *frame,
                       xlator_t *this,
@@ -611,20 +676,29 @@ int32_t dht_xattrop (call_frame_t           *frame,
                      xlator_t           *this,
                      loc_t              *loc,
                      gf_xattrop_flags_t  flags,
-                     dict_t             *dict);
+                     dict_t             *dict, dict_t *xdata);
 
 int32_t dht_fxattrop (call_frame_t *frame,
                       xlator_t           *this,
                       fd_t               *fd,
                       gf_xattrop_flags_t  flags,
-                      dict_t             *dict);
+                      dict_t             *dict, dict_t *xdata);
 
 int32_t dht_forget (xlator_t *this, inode_t *inode);
 int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
-                     struct iatt   *stbuf, int32_t valid);
+                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
 int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      struct iatt  *stbuf, int32_t valid);
-
+                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
+int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
+int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    off_t offset, size_t len, dict_t *xdata);
+int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    off_t offset, off_t len, dict_t *xdata);
+
+int32_t dht_init (xlator_t *this);
+void    dht_fini (xlator_t *this);
+int     dht_reconfigure (xlator_t *this, dict_t *options);
 int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
 
 /* definitions for nufa/switch */
@@ -647,17 +721,19 @@ int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int op_ret, int op_errno,
                     fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                    struct iatt *preparent, struct iatt *postparent);
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata);
 int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int op_ret, int op_errno,
                      inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                     struct iatt *postparent);
+                     struct iatt *postparent, dict_t *xdata);
 
 int
 gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output);
+gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
+                dict_t *output);
 
 void*
 gf_defrag_start (void *this);
@@ -668,4 +744,43 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
 int
 dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                  int flag);
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
+                          dht_layout_t **layout_int);
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t* layout_int);
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t update_ctx);
+
+int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
+int
+dht_dir_attr_heal (void *data);
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+int
+dht_dir_has_layout (dict_t *xattr, char *name);
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
+xlator_t *
+dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol,
+                                   dht_layout_t *layout);
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout);
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix);
+int32_t
+dht_priv_dump (xlator_t *this);
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode);
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol);
+
 #endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index 0200695d2..fe3955ecb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -35,7 +26,8 @@
 
 int
 dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, struct statvfs *statvfs)
+		 int op_ret, int op_errno, struct statvfs *statvfs,
+                 dict_t *xdata)
 {
 	dht_conf_t    *conf         = NULL;
 	call_frame_t  *prev          = NULL;
@@ -129,7 +121,7 @@ dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
 	STACK_WIND (statfs_frame, dht_du_info_cbk,
 		    conf->subvolumes[subvol_idx],
 		    conf->subvolumes[subvol_idx]->fops->statfs,
-		    &tmp_loc);
+		    &tmp_loc, NULL);
 
 	return 0;
 err:
@@ -177,7 +169,7 @@ dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
 			STACK_WIND (statfs_frame, dht_du_info_cbk,
 				    conf->subvolumes[i],
 				    conf->subvolumes[i]->fops->statfs,
-				    &tmp_loc);
+				    &tmp_loc, NULL);
 		}
 
 		conf->last_stat_fetch.tv_sec = tv.tv_sec;
@@ -256,50 +248,167 @@ dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
 	return is_subvol_filled;
 }
 
+
+/*Get the best subvolume to create the file in*/
 xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol)
+dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                dht_local_t *local)
 {
-	int         i = 0;
-	double      max = 0;
-	double      max_inodes = 0;
 	xlator_t   *avail_subvol = NULL;
 	dht_conf_t *conf = NULL;
+        dht_layout_t *layout = NULL;
+        loc_t      *loc = NULL;
 
 	conf = this->private;
-
-	LOCK (&conf->subvolume_lock);
+        if (!local)
+                goto out;
+        loc = &local->loc;
+        if (!local->layout) {
+                layout = dht_layout_get (this, loc->parent);
+
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "layout missing path=%s parent=%s",
+                                loc->path, uuid_utoa (loc->parent->gfid));
+                        goto out;
+                }
+        } else {
+                layout = dht_layout_ref (this, local->layout);
+        }
+
+        LOCK (&conf->subvolume_lock);
 	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (conf->disk_unit == 'p') {
-				if ((conf->du_stats[i].avail_percent > max)
-				    && (conf->du_stats[i].avail_inodes > max_inodes)) {
-					max = conf->du_stats[i].avail_percent;
-					max_inodes = conf->du_stats[i].avail_inodes;
-					avail_subvol = conf->subvolumes[i];
-				}
-			} else {
-				if ((conf->du_stats[i].avail_space > max)
-				    && (conf->du_stats[i].avail_inodes > max_inodes)) {
-					max = conf->du_stats[i].avail_space;
-					max_inodes = conf->du_stats[i].avail_inodes;
-					avail_subvol = conf->subvolumes[i];
-				}
+                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol,
+                                                                 layout);
+                if(!avail_subvol)
+                {
+                        avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
+                                                                        subvol,
+                                                                        layout);
+                }
 
-			}
-		}
 	}
 	UNLOCK (&conf->subvolume_lock);
-
+out:
 	if (!avail_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume has enough free space and inodes to create");
+		gf_log (this->name,
+                        GF_LOG_DEBUG,
+			"no subvolume has enough free space and/or inodes\
+                         to create");
+                avail_subvol = subvol;
 	}
 
-	if ((max < conf->min_free_disk) && (max_inodes < conf->min_free_inodes))
-		avail_subvol = subvol;
+        if (layout)
+                dht_layout_unref (this, layout);
+	return avail_subvol;
+}
 
-	if (!avail_subvol)
-		avail_subvol = subvol;
+static inline
+int32_t dht_subvol_has_err (xlator_t *this, dht_layout_t *layout)
+{
+        int ret = -1;
+        int i   = 0;
+
+        if (!this || !layout)
+                goto out;
+
+        /* check if subvol has layout errors, before selecting it */
+        for (i = 0; i < layout->cnt; i++) {
+                if (!strcmp (layout->list[i].xlator->name, this->name) &&
+                     (layout->list[i].err != 0)) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
 
-	return avail_subvol;
+/*Get subvolume which has both space and inodes more than the min criteria*/
+xlator_t *
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int i = 0;
+        double max = 0;
+        double max_inodes = 0;
+        int    ignore_subvol = 0;
+
+        xlator_t *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        for(i=0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if ((conf->disk_unit == 'p') &&
+                    (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_percent > max)) {
+                                max = conf->du_stats[i].avail_percent;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
+
+                if ((conf->disk_unit != 'p') &&
+                    (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_space > max)) {
+                                max = conf->du_stats[i].avail_space;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
+        }
+
+        return avail_subvol;
+}
+
+
+/* Get subvol which has atleast one inode and maximum space */
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int         i = 0;
+        double      max = 0;
+        int         ignore_subvol = 0;
+
+        xlator_t   *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if (conf->disk_unit == 'p') {
+                        if ((conf->du_stats[i].avail_percent > max)
+                            && (conf->du_stats[i].avail_inodes > 0 )) {
+                                max = conf->du_stats[i].avail_percent;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+               } else {
+                         if ((conf->du_stats[i].avail_space > max)
+                            && (conf->du_stats[i].avail_inodes > 0)) {
+                                 max = conf->du_stats[i].avail_space;
+                                 avail_subvol = conf->subvolumes[i];
+                         }
+               }
+        }
+
+        return avail_subvol;
 }
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index c8ae74172..656cf23a0 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -37,6 +28,7 @@ dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 
         switch (type) {
         case DHT_HASH_TYPE_DM:
+        case DHT_HASH_TYPE_DM_USER:
                 hash = gf_dm_hashfn (name, strlen (name));
                 break;
         default:
@@ -52,30 +44,68 @@ dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 }
 
 
-#define MAKE_RSYNC_FRIENDLY_NAME(rsync_frndly_name, name) do {          \
-                rsync_frndly_name = (char *) name;                      \
-                if (name[0] == '.') {                                   \
-                        char *dot   = 0;                                \
-                        int namelen = 0;                                \
-                                                                        \
-                        dot = strrchr (name, '.');                      \
-                        if (dot && dot > (name + 1) && *(dot + 1)) {    \
-                                namelen = (dot - name);                 \
-                                rsync_frndly_name = alloca (namelen);   \
-                                strncpy (rsync_frndly_name, name + 1,   \
-                                         namelen);                      \
-                                rsync_frndly_name[namelen - 1] = 0;     \
-                        }                                               \
-                }                                                       \
-        } while (0);
+static inline
+gf_boolean_t
+dht_munge_name (const char *original, char *modified, size_t len, regex_t *re)
+{
+        regmatch_t      matches[2];
+        size_t          new_len;
+
+        if (regexec(re,original,2,matches,0) != REG_NOMATCH) {
+                if (matches[1].rm_so != -1) {
+                        new_len = matches[1].rm_eo - matches[1].rm_so;
+                        /* Equal would fail due to the NUL at the end. */
+                        if (new_len < len) {
+                                memcpy (modified,original+matches[1].rm_so,
+                                        new_len);
+                                modified[new_len] = '\0';
+                                return _gf_true;
+                        }
+                }
+        }
 
+        /* This is guaranteed safe because of how the dest was allocated. */
+        strcpy(modified,original);
+        return _gf_false;
+}
 
 int
-dht_hash_compute (int type, const char *name, uint32_t *hash_p)
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-        char     *rsync_friendly_name = NULL;
+        char            *rsync_friendly_name    = NULL;
+        dht_conf_t      *priv                   = this->private;
+        size_t           len                    = 0;
+        gf_boolean_t     munged                 = _gf_false;
+
+        /*
+         * It wouldn't be safe to use alloca in an inline function that doesn't
+         * actually get inlined, and it wouldn't be efficient to do a real
+         * allocation, so we use alloca here (if needed) and pass that to the
+         * inline.
+         */
+
+        if (priv->extra_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->extra_regex);
+        }
+
+        if (!munged && priv->rsync_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                gf_log (this->name, GF_LOG_TRACE, "trying regex for %s", name);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->rsync_regex);
+                if (munged) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "munged down to %s", rsync_friendly_name);
+                }
+        }
 
-        MAKE_RSYNC_FRIENDLY_NAME (rsync_friendly_name, name);
+        if (!munged) {
+                rsync_friendly_name = (char *)name;
+        }
 
         return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index ff540013e..18a501f04 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -27,6 +18,28 @@
 #include "xlator.h"
 #include "dht-common.h"
 
+static inline int
+dht_inode_ctx_set1 (xlator_t *this, inode_t *inode, xlator_t *subvol)
+{
+        uint64_t tmp_subvol = 0;
+
+        tmp_subvol = (long)subvol;
+        return inode_ctx_set1 (inode, this, &tmp_subvol);
+}
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol)
+{
+        int ret = -1;
+        uint64_t tmp_subvol = 0;
+
+        ret =  inode_ctx_get1 (inode, this, &tmp_subvol);
+        if (tmp_subvol && subvol)
+                *subvol = (xlator_t *)tmp_subvol;
+
+        return ret;
+}
+
 
 int
 dht_frame_return (call_frame_t *frame)
@@ -49,6 +62,43 @@ dht_frame_return (call_frame_t *frame)
 }
 
 
+static uint64_t
+dht_bits_for (uint64_t num)
+{
+	uint64_t bits = 0, ctrl = 1;
+
+	while (ctrl < num) {
+		ctrl *= 2;
+		bits ++;
+	}
+
+	return bits;
+}
+
+/*
+ * A slightly "updated" version of the algorithm described in the commit log
+ * is used here.
+ *
+ * The only enhancement is that:
+ *
+ * - The number of bits used by the backend filesystem for HUGE d_off which
+ *   is described as 63, and
+ * - The number of bits used by the d_off presented by the transformation
+ *   upwards which is described as 64, are both made "configurable."
+ */
+
+
+#define BACKEND_D_OFF_BITS 63
+#define PRESENT_D_OFF_BITS 63
+
+#define ONE 1ULL
+#define MASK (~0ULL)
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
+#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS))
+
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
+
 int
 dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
 {
@@ -56,6 +106,9 @@ dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
         int         cnt = 0;
         int         max = 0;
         uint64_t    y = 0;
+        uint64_t    hi_mask = 0;
+        uint64_t    off_mask = 0;
+        int         max_bits = 0;
 
         if (x == ((uint64_t) -1)) {
                 y = (uint64_t) -1;
@@ -69,7 +122,23 @@ dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
         max = conf->subvolume_cnt;
         cnt = dht_subvol_cnt (this, subvol);
 
-        y = ((x * max) + cnt);
+	if (max == 1) {
+		y = x;
+		goto out;
+	}
+
+        max_bits = dht_bits_for (max);
+
+        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
+
+        if (x & hi_mask) {
+                /* HUGE d_off */
+                off_mask = MASK << max_bits;
+                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | cnt;
+        } else {
+                /* small d_off */
+                y = ((x * max) + cnt);
+        }
 
 out:
         if (y_p)
@@ -89,7 +158,7 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
         int            ret       = 0; /* not found */
 
         /* Why do other tasks if first required 'char' itself is not there */
-        if (!loc->name || !strchr (loc->name, '@'))
+        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
                 goto out;
 
         trav = this->children;
@@ -129,10 +198,8 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
 out:
         if (!ret) {
                 /* !success */
-                if (new_path)
-                        GF_FREE (new_path);
-                if (new_name)
-                        GF_FREE (new_name);
+                GF_FREE (new_path);
+                GF_FREE (new_name);
         }
         return ret;
 }
@@ -146,16 +213,38 @@ dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
         int         max = 0;
         uint64_t    x = 0;
         xlator_t   *subvol = 0;
+        int         max_bits = 0;
+        uint64_t    off_mask = 0;
+        uint64_t    host_mask = 0;
 
         if (!this->private)
-                goto out;
+                return -1;
 
         conf = this->private;
         max = conf->subvolume_cnt;
 
-        cnt = y % max;
-        x   = y / max;
+	if (max == 1) {
+		x = y;
+		cnt = 0;
+		goto out;
+	}
+
+        if (y & TOP_BIT) {
+                /* HUGE d_off */
+                max_bits = dht_bits_for (max);
+                off_mask = (MASK << max_bits);
+                host_mask = ~(off_mask);
+
+                x = ((y & ~TOP_BIT) & off_mask) << SHIFT_BITS;
+
+                cnt = y & host_mask;
+	} else {
+                /* small d_off */
+                cnt = y % max;
+                x = y / max;
+        }
 
+out:
         subvol = conf->subvolumes[cnt];
 
         if (subvol_p)
@@ -164,7 +253,6 @@ dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
         if (x_p)
                 *x_p = x;
 
-out:
         return 0;
 }
 
@@ -215,16 +303,11 @@ dht_local_wipe (xlator_t *this, dht_local_t *local)
                 local->selfheal.layout = NULL;
         }
 
-        if (local->newpath) {
-                GF_FREE (local->newpath);
-        }
+        GF_FREE (local->newpath);
 
-        if (local->key) {
-                GF_FREE (local->key);
-        }
+        GF_FREE (local->key);
 
-        if (local->rebalance.vector)
-                GF_FREE (local->rebalance.vector);
+        GF_FREE (local->rebalance.vector);
 
         if (local->rebalance.iobref)
                 iobref_unref (local->rebalance.iobref);
@@ -279,20 +362,6 @@ out:
         return local;
 }
 
-
-char *
-basestr (const char *str)
-{
-        char *basestr = NULL;
-
-        basestr = strrchr (str, '/');
-        if (basestr)
-                basestr ++;
-
-        return basestr;
-}
-
-
 xlator_t *
 dht_first_up_subvol (xlator_t *this)
 {
@@ -444,7 +513,36 @@ out:
         return next;
 }
 
+/* This func wraps around, if prev is actually the last subvol.
+ */
+xlator_t *
+dht_subvol_next_available (xlator_t *this, xlator_t *prev)
+{
+        dht_conf_t *conf = NULL;
+        int         i = 0;
+        xlator_t   *next = NULL;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == prev) {
+                        /* if prev is last in conf->subvolumes, then wrap
+                         * around.
+                         */
+                        if ((i + 1) < conf->subvolume_cnt) {
+                                next = conf->subvolumes[i + 1];
+                        } else {
+                                next = conf->subvolumes[0];
+                        }
+                        break;
+                }
+        }
 
+out:
+        return next;
+}
 int
 dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
 {
@@ -636,36 +734,61 @@ dht_migration_complete_check_task (void *data)
         call_frame_t *frame    = NULL;
         loc_t         tmp_loc  = {0,};
         char         *path     = NULL;
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        uint64_t      tmp_subvol = 0;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
         local = frame->local;
+        conf = this->private;
 
         src_node = local->cached_subvol;
 
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (!local->loc.inode)
+        if (!local->loc.inode && !local->fd) {
+                local->op_errno = EINVAL;
+                goto out;
+        }
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+
+        if (!local->loc.inode) {
                 ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
-        else
+                                        conf->link_xattr_name);
+        } else {
+                SYNCTASK_SETID (0, 0);
                 ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        }
 
         if (!ret)
                 dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
 
         if (ret) {
-                if ((errno != ENOENT) || (!local->loc.inode)) {
+                if (!dht_inode_missing(-ret) || (!local->loc.inode)) {
+                        local->op_errno = -ret;
                         gf_log (this->name, GF_LOG_ERROR,
                                 "%s: failed to get the 'linkto' xattr %s",
-                                local->loc.path, strerror (errno));
+                                local->loc.path, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
+
                 /* Need to do lookup on hashed subvol, then get the file */
                 ret = syncop_lookup (this, &local->loc, NULL, &stbuf, NULL,
                                      NULL);
-                if (ret)
+                if (ret) {
+                        local->op_errno = -ret;
+                        ret = -1;
                         goto out;
+                }
+
                 dst_node = dht_subvol_get_cached (this, local->loc.inode);
         }
 
@@ -674,17 +797,21 @@ dht_migration_complete_check_task (void *data)
                         "%s: failed to get the destination node",
                         local->loc.path);
                 ret = -1;
+                local->op_errno = EINVAL;
                 goto out;
         }
 
         /* lookup on dst */
         if (local->loc.inode) {
-                ret = syncop_lookup (dst_node, &local->loc, NULL, &stbuf, NULL, NULL);
+                ret = syncop_lookup (dst_node, &local->loc, NULL, &stbuf, NULL,
+                                     NULL);
 
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "%s: failed to lookup the file on %s",
                                 local->loc.path, dst_node->name);
+                        local->op_errno = -ret;
+                        ret = -1;
                         goto out;
                 }
 
@@ -693,6 +820,7 @@ dht_migration_complete_check_task (void *data)
                                 "%s: gfid different on the target file on %s",
                                 local->loc.path, dst_node->name);
                         ret = -1;
+                        local->op_errno = EIO;
                         goto out;
                 }
         }
@@ -700,15 +828,13 @@ dht_migration_complete_check_task (void *data)
         /* update inode ctx (the layout) */
         dht_layout_unref (this, local->layout);
 
-        if (!local->loc.inode)
-                ret = dht_layout_preset (this, dst_node, local->fd->inode);
-        else
-                ret = dht_layout_preset (this, dst_node, local->loc.inode);
+        ret = dht_layout_preset (this, dst_node, inode);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "%s: could not set preset layout for subvol %s",
                         local->loc.path, dst_node->name);
                 ret   = -1;
+                local->op_errno = EINVAL;
                 goto out;
         }
 
@@ -718,60 +844,69 @@ dht_migration_complete_check_task (void *data)
                         "%s: no pre-set layout for subvolume %s",
                         local->loc.path, dst_node ? dst_node->name : "<nil>");
                 ret = -1;
+                local->op_errno = EINVAL;
                 goto out;
         }
 
-        if (!local->loc.inode)
-                ret = dht_layout_set (this, local->fd->inode, layout);
-        else
-                ret = dht_layout_set (this, local->loc.inode, layout);
+        ret = dht_layout_set (this, inode, layout);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set the new layout",
                         local->loc.path);
+                local->op_errno = EINVAL;
                 goto out;
         }
 
         local->cached_subvol = dst_node;
         ret = 0;
 
-        if (!local->fd)
+        /* once we detect the migration complete, the inode-ctx2 is no more
+           required.. delete the ctx and also, it means, open() already
+           done on all the fd of inode */
+        ret = inode_ctx_reset1 (inode, this, &tmp_subvol);
+        if (tmp_subvol)
                 goto out;
 
-        /* once we detect the migration complete, the fd-ctx is no more
-           required.. delete the ctx, and do one extra 'fd_unref' for open fd */
-        ret = fd_ctx_del (local->fd, this, NULL);
-        if (!ret) {
-                fd_unref (local->fd);
-                ret = 0;
+        if (list_empty (&inode->fd_list))
                 goto out;
-        }
 
-        /* if 'local->fd' (ie, fd based operation), send a 'open()' on
-           destination if not already done */
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID(0, 0);
+
+        /* perform 'open()' on all the fd's present on the inode */
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                /* flags for open are stripped down to allow following the
+                 * new location of the file, otherwise we can get EEXIST or
+                 * truncate the file again as rebalance is moving the data */
                 ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                if (path)
-                        GF_FREE (path);
-
+                                   (iter_fd->flags &
+                                   ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                        local->op_errno = -ret;
+                        ret = -1;
+                }
         }
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        GF_FREE (path);
+
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
-
-        /* need this unref for the fd on src_node */
-        fd_unref (local->fd);
         ret = 0;
 out:
 
@@ -782,11 +917,8 @@ int
 dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
 {
         int         ret     = -1;
-        dht_conf_t *conf    = NULL;
 
-        conf = this->private;
-
-        ret = synctask_new (conf->env, dht_migration_complete_check_task,
+        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
                             dht_migration_complete_check_done,
                             frame, frame);
         return ret;
@@ -818,26 +950,41 @@ dht_rebalance_inprogress_task (void *data)
         char         *path     = NULL;
         struct iatt   stbuf    = {0,};
         loc_t         tmp_loc  = {0,};
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
         local = frame->local;
+        conf = this->private;
 
         src_node = local->cached_subvol;
 
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (local->loc.inode)
+        if (!local->loc.inode && !local->fd)
+                goto out;
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+        if (local->loc.inode) {
+                SYNCTASK_SETID (0, 0);
                 ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
-        else
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        } else {
                 ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
+                                        conf->link_xattr_name);
+        }
 
-        if (ret) {
+        if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to get the 'linkto' xattr %s",
-                        local->loc.path, strerror (errno));
-                        goto out;
+                        local->loc.path, strerror (-ret));
+                ret = -1;
+                goto out;
         }
 
         dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
@@ -859,6 +1006,7 @@ dht_rebalance_inprogress_task (void *data)
                         gf_log (this->name, GF_LOG_ERROR,
                                 "%s: failed to lookup the file on %s",
                                 local->loc.path, dst_node->name);
+                        ret = -1;
                         goto out;
                 }
 
@@ -873,34 +1021,51 @@ dht_rebalance_inprogress_task (void *data)
 
         ret = 0;
 
-        if (!local->fd)
-                goto out;
+        if (list_empty (&inode->fd_list))
+                goto done;
 
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID (0, 0);
+
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                /* flags for open are stripped down to allow following the
+                 * new location of the file, otherwise we can get EEXIST or
+                 * truncate the file again as rebalance is moving the data */
                 ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                if (path)
-                        GF_FREE (path);
+                                   (iter_fd->flags &
+                                   ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to send open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        ret = -1;
+                        open_failed = 1;
+                }
         }
+        GF_FREE (path);
 
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
 
-        ret = fd_ctx_set (local->fd, this, (uint64_t)(long)dst_node);
+done:
+        ret = dht_inode_ctx_set1 (this, inode, dst_node);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set fd-ctx target file at %s",
+                        "%s: failed to set inode-ctx target file at %s",
                         local->loc.path, dst_node->name);
                 goto out;
         }
@@ -915,12 +1080,99 @@ dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
 {
 
         int         ret     = -1;
-        dht_conf_t *conf    = NULL;
 
-        conf = this->private;
-
-        ret = synctask_new (conf->env, dht_rebalance_inprogress_task,
+        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
                             dht_inprogress_check_done,
                             frame, frame);
         return ret;
 }
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+        if (!ret && ctx) {
+                ctx->layout = layout_int;
+        } else {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return ret;
+                ctx->layout = layout_int;
+        }
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+
+        return ret;
+}
+
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t post)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        dht_stat_time_t         *time           = 0;
+        int                      ret            = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, stat, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (ret) {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return -1;
+        }
+
+        time = &ctx->time;
+
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
+                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
+                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
+                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+out:
+        return 0;
+}
+
+int
+dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+{
+        int             ret     = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+
+        if (ret)
+                return ret;
+
+        if (ctx)
+                *ctx = (dht_inode_ctx_t *) ctx_int;
+out:
+        return ret;
+}
+
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
+{
+        int             ret = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ctx_int = (long)ctx;
+        ret = inode_ctx_set (inode, this, &ctx_int);
+out:
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 3ec6f4ba2..e8a9a7196 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -34,7 +25,7 @@ int dht_fsync2 (xlator_t *this, call_frame_t *frame, int ret);
 
 int
 dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, fd_t *fd)
+              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -44,7 +35,7 @@ dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         local->op_errno = op_errno;
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
                         prev->this->name, strerror (op_errno));
@@ -61,7 +52,7 @@ dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd);
+        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
 
         return 0;
 }
@@ -86,17 +77,17 @@ dht_open2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
                     &local->loc, local->rebalance.flags, local->fd,
-                    local->rebalance.wbflags);
+                    NULL);
         return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int
 dht_open (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, int flags, fd_t *fd, int wbflags)
+          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -120,30 +111,30 @@ dht_open (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
-        local->rebalance.wbflags = wbflags;
         local->rebalance.flags = flags;
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf)
+                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        uint64_t      tmp_subvol = 0;
+        xlator_t     *subvol = 0;
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -153,7 +144,7 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = frame->local;
         prev = cookie;
 
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
                 local->op_errno = op_errno;
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
@@ -164,26 +155,28 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->call_cnt != 1)
                 goto out;
 
+        local->op_errno = op_errno;
         /* Check if the rebalance phase2 is true */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                if (local->fd)
-                        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-                if (ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         /* Phase 2 of migration */
                         local->rebalance.target_op_fn = dht_attr2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_attr2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf);
+        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
 err:
         return 0;
 }
@@ -208,21 +201,21 @@ dht_attr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         if (local->fop == GF_FOP_FSTAT) {
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, local->fd);
+                            subvol->fops->fstat, local->fd, NULL);
         } else {
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, &local->loc);
+                            subvol->fops->stat, &local->loc, NULL);
         }
         return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int
 dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, struct iatt *stbuf)
+              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -257,14 +250,14 @@ out:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
                 DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf);
+                                  &local->stbuf, xdata);
         }
 err:
         return 0;
 }
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -300,7 +293,7 @@ dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 subvol = local->cached_subvol;
 
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, loc);
+                            subvol->fops->stat, loc, xdata);
 
                 return 0;
         }
@@ -312,21 +305,21 @@ dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
                 STACK_WIND (frame, dht_attr_cbk,
                             subvol, subvol->fops->stat,
-                            loc);
+                            loc, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -360,7 +353,7 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 subvol = local->cached_subvol;
 
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, fd);
+                            subvol->fops->fstat, fd, xdata);
 
                 return 0;
         }
@@ -371,14 +364,14 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 subvol = layout->list[i].xlator;
                 STACK_WIND (frame, dht_attr_cbk,
                             subvol, subvol->fops->fstat,
-                            fd);
+                            fd, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -387,10 +380,12 @@ int
 dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno,
                struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref)
+               struct iobref *iobref, dict_t *xdata)
 {
         dht_local_t *local      = NULL;
         int          ret        = 0;
+        inode_t     *inode      = NULL;
+        xlator_t    *subvol = 0;
 
         local = frame->local;
         if (!local) {
@@ -403,28 +398,30 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->call_cnt != 1)
                 goto out;
 
-        if ((op_ret == -1) && (op_errno != ENOENT))
+        if ((op_ret == -1) && !dht_inode_missing(op_errno))
                 goto out;
 
+        local->op_errno = op_errno;
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 /* File would be migrated to other node */
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (ret) {
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         local->rebalance.target_op_fn = dht_readv2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_readv2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                          iobref);
+                          iobref, xdata);
 
         return 0;
 }
@@ -449,18 +446,18 @@ dht_readv2 (xlator_t *this, call_frame_t *frame, int op_ret)
 
         STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
                     local->fd, local->rebalance.size, local->rebalance.offset,
-                    local->rebalance.flags);
+                    local->rebalance.flags, NULL);
 
         return 0;
 
 out:
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
         return 0;
 }
 
 int
 dht_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags)
+           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -491,31 +488,52 @@ dht_readv (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_readv_cbk,
                     subvol, subvol->fops->readv,
-                    fd, size, off, flags);
+                    fd, size, off, flags, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
         return 0;
 }
 
 int
 dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno)
+                int op_ret, int op_errno, dict_t *xdata)
 {
         int          ret = -1;
         dht_local_t *local = NULL;
+        xlator_t    *subvol = NULL;
+        call_frame_t *prev = NULL;
 
         local = frame->local;
+        prev = cookie;
 
+        if (!prev || !prev->this)
+                goto out;
         if (local->call_cnt != 1)
                 goto out;
+        if ((op_ret == -1) && (op_errno == ENOTCONN) &&
+            IA_ISDIR(local->loc.inode->ia_type)) {
+
+                subvol = dht_subvol_next_available (this, prev->this);
+                if (!subvol)
+                        goto out;
 
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                /* check if we are done with visiting every node */
+                if (subvol == local->cached_subvol) {
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                            &local->loc, local->rebalance.flags, NULL);
+                return 0;
+        }
+        if ((op_ret == -1) && dht_inode_missing(op_errno)) {
                 /* File would be migrated to other node */
+                local->op_errno = op_errno;
                 local->rebalance.target_op_fn = dht_access2;
                 ret = dht_rebalance_complete_check (frame->this, frame);
                 if (!ret)
@@ -523,7 +541,7 @@ dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 out:
-        DHT_STACK_UNWIND (access, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -546,18 +564,19 @@ dht_access2 (xlator_t *this, call_frame_t *frame, int op_ret)
         subvol = local->cached_subvol;
 
         STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    &local->loc, local->rebalance.flags);
+                    &local->loc, local->rebalance.flags, NULL);
 
         return 0;
 
 out:
-        DHT_STACK_UNWIND (access, frame, -1, op_errno);
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int
-dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -586,13 +605,13 @@ dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
         }
 
         STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (access, frame, -1, op_errno);
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -600,10 +619,11 @@ err:
 
 int
 dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno)
+               int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           ret = -1;
+        dht_local_t  *local  = NULL;
+        inode_t      *inode  = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
 
@@ -613,14 +633,14 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
 
         /* If context is set, then send flush() it to the destination */
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (!ret) {
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (subvol) {
                 dht_flush2 (this, frame, 0);
                 return 0;
         }
 
 out:
-        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -630,14 +650,10 @@ dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -645,20 +661,19 @@ dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
         local->call_cnt = 2; /* This is the second attempt */
 
         STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, local->fd);
+                    subvol, subvol->fops->flush, local->fd, NULL);
 
         return 0;
 }
 
 
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
 
-
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
@@ -680,13 +695,13 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
         local->call_cnt = 1;
 
         STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, fd);
+                    subvol, subvol->fops->flush, fd, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno);
+        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -694,17 +709,20 @@ err:
 
 int
 dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf)
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
         prev = cookie;
 
         local->op_errno = op_errno;
-        if (op_ret == -1) {
+        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
                         prev->this->name, strerror (op_errno));
@@ -719,8 +737,9 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 goto out;
         }
 
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (ret) {
+        local->op_errno = op_errno;
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol) {
                 local->rebalance.target_op_fn = dht_fsync2;
 
                 /* Check if the rebalance phase1 is true */
@@ -735,17 +754,18 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                         ret = dht_rebalance_complete_check (this, frame);
                 }
+                if (!ret)
+                        return 0;
         } else {
                 dht_fsync2 (this, frame, 0);
-        }
-        if (!ret)
                 return 0;
+        }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
-                          prebuf, postbuf);
+                          prebuf, postbuf, xdata);
 
         return 0;
 }
@@ -755,34 +775,29 @@ dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
-
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
         local->call_cnt = 2; /* This is the second attempt */
 
         STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    local->fd, local->rebalance.flags);
+                    local->fd, local->rebalance.flags, NULL);
 
         return 0;
 }
 
 int
-dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+           dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
 
-
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
@@ -800,13 +815,13 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
         subvol = local->cached_subvol;
 
         STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    fd, datasync);
+                    fd, datasync, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -817,9 +832,9 @@ err:
    phase 2 of migration */
 int
 dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, struct gf_flock *flock)
+            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock);
+        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
 
         return 0;
 }
@@ -827,7 +842,7 @@ dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int cmd, struct gf_flock *flock)
+        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -847,13 +862,13 @@ dht_lk (call_frame_t *frame, xlator_t *this,
 
         /* TODO: for rebalance, we need to preserve the fop arguments */
         STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, fd,
-                    cmd, flock);
+                    cmd, flock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -861,7 +876,8 @@ err:
 /* Symlinks are currently not migrated, so no need for any check here */
 int
 dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, const char *path, struct iatt *stbuf)
+                  int op_ret, int op_errno, const char *path,
+                  struct iatt *stbuf, dict_t *xdata)
 {
         dht_local_t *local = NULL;
 
@@ -876,14 +892,15 @@ dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 err:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf);
+        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
 
         return 0;
 }
 
 
 int
-dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+              dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -911,13 +928,13 @@ dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
 
         STACK_WIND (frame, dht_readlink_cbk,
                     subvol, subvol->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -928,16 +945,16 @@ err:
 
 int
 dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict)
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t flags, dict_t *dict)
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -968,13 +985,13 @@ dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         STACK_WIND (frame,
                     dht_xattrop_cbk,
                     subvol, subvol->fops->xattrop,
-                    loc, flags, dict);
+                    loc, flags, dict, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -982,16 +999,16 @@ err:
 
 int
 dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict);
+        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 dht_fxattrop (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1011,13 +1028,13 @@ dht_fxattrop (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame,
                     dht_fxattrop_cbk,
                     subvol, subvol->fops->fxattrop,
-                    fd, flags, dict);
+                    fd, flags, dict, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL);
+        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -1025,17 +1042,17 @@ err:
 
 int
 dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
+dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1067,31 +1084,31 @@ dht_inodelk (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame,
                     dht_inodelk_cbk,
                     subvol, subvol->fops->inodelk,
-                    volume, loc, cmd, lock);
+                    volume, loc, cmd, lock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
 
         return 0;
 }
 
 
 int
-dht_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
-dht_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -1109,16 +1126,14 @@ dht_finodelk (call_frame_t *frame, xlator_t *this,
         }
 
 
-        STACK_WIND (frame,
-                    dht_finodelk_cbk,
-                    subvol, subvol->fops->finodelk,
-                    volume, fd, cmd, lock);
+        STACK_WIND (frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno);
+        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 2f59e7c94..363bff3bf 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -28,16 +19,20 @@
 int dht_writev2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret);
 
 int
 dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+                struct iatt *postbuf, dict_t *xdata)
 {
         dht_local_t *local = NULL;
         int          ret   = -1;
+        xlator_t    *subvol = NULL;
 
-        if (op_ret == -1) {
+        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
                 goto out;
         }
 
@@ -59,6 +54,7 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_writev2;
 
+        local->op_errno = op_errno;
         /* Phase 2 of migration */
         if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -71,8 +67,8 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
 
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                ret = dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
                         dht_writev2 (this, frame, 0);
                         return 0;
                 }
@@ -85,7 +81,8 @@ out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
 
-        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
@@ -95,14 +92,10 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -113,7 +106,7 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
                     subvol, subvol->fops->writev,
                     local->fd, local->rebalance.vector, local->rebalance.count,
                     local->rebalance.offset, local->rebalance.flags,
-                    local->rebalance.iobref);
+                    local->rebalance.iobref, NULL);
 
         return 0;
 }
@@ -121,7 +114,7 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
 int
 dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int count, off_t off, uint32_t flags,
-            struct iobref *iobref)
+            struct iobref *iobref, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -156,13 +149,13 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         STACK_WIND (frame, dht_writev_cbk,
                     subvol, subvol->fops->writev,
-                    fd, vector, count, off, flags, iobref);
+                    fd, vector, count, off, flags, iobref, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -172,11 +165,13 @@ err:
 int
 dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        xlator_t    *subvol = NULL;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -186,7 +181,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = frame->local;
         prev = cookie;
 
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
                 local->op_errno = op_errno;
                 local->op_ret = -1;
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -206,6 +201,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_truncate2;
 
+        local->op_errno = op_errno;
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -217,8 +213,9 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
                         dht_truncate2 (this, frame, 0);
                         return 0;
                 }
@@ -231,7 +228,7 @@ out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                          prebuf, postbuf);
+                          prebuf, postbuf, xdata);
 err:
         return 0;
 }
@@ -242,16 +239,13 @@ dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = local->fd ? local->fd->inode : local->loc.inode;
 
+        dht_inode_ctx_get1 (this, inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
@@ -260,18 +254,19 @@ dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
         if (local->fop == GF_FOP_TRUNCATE) {
                 STACK_WIND (frame, dht_truncate_cbk, subvol,
                             subvol->fops->truncate, &local->loc,
-                            local->rebalance.offset);
+                            local->rebalance.offset, NULL);
         } else {
                 STACK_WIND (frame, dht_truncate_cbk, subvol,
                             subvol->fops->ftruncate, local->fd,
-                            local->rebalance.offset);
+                            local->rebalance.offset, NULL);
         }
 
         return 0;
 }
 
 int
-dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -301,19 +296,20 @@ dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
 
         STACK_WIND (frame, dht_truncate_cbk,
                     subvol, subvol->fops->truncate,
-                    loc, offset);
+                    loc, offset, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
 
 int
-dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
@@ -341,22 +337,423 @@ dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
 
         STACK_WIND (frame, dht_truncate_cbk,
                     subvol, subvol->fops->ftruncate,
-                    fd, offset);
+                    fd, offset, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_fallocate2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_fallocate2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_fallocate_cbk, subvol, subvol->fops->fallocate,
+		   local->fd, local->rebalance.flags, local->rebalance.offset,
+		   local->rebalance.size, NULL);
+
+        return 0;
+}
+
+int
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	      off_t offset, size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FALLOCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+	local->rebalance.flags = mode;
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_fallocate_cbk,
+                    subvol, subvol->fops->fallocate,
+                    fd, mode, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_discard2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_discard2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_discard_cbk, subvol, subvol->fops->discard,
+		   local->fd, local->rebalance.offset, local->rebalance.size,
+		   NULL);
+
+        return 0;
+}
+
+int
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_DISCARD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_discard_cbk, subvol, subvol->fops->discard,
+                    fd, offset, len, xdata);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
 
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev  = NULL;
+        int           ret   = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_zerofill2;
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                ret = fd_ctx_get (local->fd, this, NULL);
+                if (!ret) {
+                        dht_zerofill2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local          = NULL;
+        xlator_t     *subvol         = NULL;
+        uint64_t      tmp_subvol     = 0;
+        int           ret            = -1;
+
+        local = frame->local;
+
+        if (local->fd)
+                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
+        if (!ret)
+                subvol = (xlator_t *)(long)tmp_subvol;
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                   local->fd, local->rebalance.offset, local->rebalance.size,
+                   NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+        xlator_t     *subvol       = NULL;
+        int           op_errno     = -1;
+        dht_local_t  *local        = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
 /* handle cases of migration here for 'setattr()' calls */
 int
 dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int op_ret, int op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -366,7 +763,7 @@ dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         local->op_errno = op_errno;
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
                         prev->this->name, strerror (op_errno));
@@ -393,7 +790,7 @@ out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
         DHT_STRIP_PHASE1_FLAGS (prebuf);
         DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                          prebuf, postbuf);
+                          prebuf, postbuf, xdata);
 
         return 0;
 }
@@ -403,15 +800,13 @@ dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -421,11 +816,13 @@ dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
         if (local->fop == GF_FOP_SETATTR) {
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->setattr, &local->loc,
-                            &local->rebalance.stbuf, local->rebalance.flags);
+                            &local->rebalance.stbuf, local->rebalance.flags,
+                            NULL);
         } else {
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->fsetattr, local->fd,
-                            &local->rebalance.stbuf, local->rebalance.flags);
+                            &local->rebalance.stbuf, local->rebalance.flags,
+                            NULL);
         }
 
         return 0;
@@ -436,7 +833,7 @@ dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 int
 dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int op_ret, int op_errno, struct iatt *statpre,
-                 struct iatt *statpost)
+                 struct iatt *statpost, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -467,7 +864,7 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt))
                 DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
-                                  &local->prebuf, &local->stbuf);
+                                  &local->prebuf, &local->stbuf, xdata);
 
         return 0;
 }
@@ -475,7 +872,7 @@ unlock:
 
 int
 dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid)
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         dht_layout_t *layout = NULL;
@@ -521,7 +918,7 @@ dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, xdata);
 
                 return 0;
         }
@@ -532,14 +929,14 @@ dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 STACK_WIND (frame, dht_setattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -547,7 +944,7 @@ err:
 
 int
 dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-              int32_t valid)
+              int32_t valid, dict_t *xdata)
 {
         xlator_t     *subvol = NULL;
         dht_layout_t *layout = NULL;
@@ -592,7 +989,7 @@ dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
 
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->fsetattr,
-                            fd, stbuf, valid);
+                            fd, stbuf, valid, xdata);
 
                 return 0;
         }
@@ -603,14 +1000,14 @@ dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
                 STACK_WIND (frame, dht_setattr_cbk,
                             layout->list[i].xlator,
                             layout->list[i].xlator->fops->fsetattr,
-                            fd, stbuf, valid);
+                            fd, stbuf, valid, xdata);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index d89e15421..e1a37b77c 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -34,14 +25,12 @@
 
 #define layout_size(cnt) (layout_base_size + (cnt * layout_entry_size))
 
-
 dht_layout_t *
 dht_layout_new (xlator_t *this, int cnt)
 {
         dht_layout_t *layout = NULL;
         dht_conf_t   *conf = NULL;
 
-
         conf = this->private;
 
         layout = GF_CALLOC (1, layout_size (cnt),
@@ -59,6 +48,7 @@ dht_layout_new (xlator_t *this, int cnt)
         }
 
         layout->ref = 1;
+
 out:
         return layout;
 }
@@ -68,9 +58,7 @@ dht_layout_t *
 dht_layout_get (xlator_t *this, inode_t *inode)
 {
         dht_conf_t   *conf = NULL;
-        uint64_t      layout_int = 0;
         dht_layout_t *layout = NULL;
-        int           ret    = -1;
 
         conf = this->private;
         if (!conf)
@@ -78,9 +66,8 @@ dht_layout_get (xlator_t *this, inode_t *inode)
 
         LOCK (&conf->layout_lock);
         {
-                ret = inode_ctx_get (inode, this, &layout_int);
-                if (ret == 0) {
-                        layout = (dht_layout_t *) (unsigned long) layout_int;
+                dht_inode_ctx_layout_get (inode, this, &layout);
+                if (layout) {
                         layout->ref++;
                 }
         }
@@ -98,7 +85,6 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
         int           oldret = -1;
         int           ret = 0;
         dht_layout_t *old_layout;
-        uint64_t      old_layout_int;
 
         conf = this->private;
         if (!conf)
@@ -106,16 +92,13 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
 
         LOCK (&conf->layout_lock);
         {
-                oldret = inode_ctx_get (inode, this, &old_layout_int);
-
+                oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
                 layout->ref++;
-                ret = inode_ctx_put (inode, this, (uint64_t) (unsigned long)
-                                     layout);
+                dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
 
-        if (oldret == 0) {
-                old_layout = (dht_layout_t *) (unsigned long) old_layout_int;
+        if (!oldret) {
                 dht_layout_unref (this, old_layout);
         }
 
@@ -130,7 +113,7 @@ dht_layout_unref (xlator_t *this, dht_layout_t *layout)
         dht_conf_t  *conf = NULL;
         int          ref = 0;
 
-        if (layout->preset || !this->private)
+        if (!layout || layout->preset || !this->private)
                 return;
 
         conf = this->private;
@@ -174,7 +157,7 @@ dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
         int        ret = 0;
 
 
-        ret = dht_hash_compute (layout->type, name, &hash);
+        ret = dht_hash_compute (this, layout->type, name, &hash);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "hash computation failed for type=%d name=%s",
@@ -280,6 +263,9 @@ dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
 
         if (disk_layout_p)
                 *disk_layout_p = disk_layout;
+        else
+                GF_FREE (disk_layout);
+
         ret = 0;
 
 out:
@@ -314,9 +300,13 @@ dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
                 return -1;
         }
 
-	switch (disk_layout[1]) {
+        type = ntoh32 (disk_layout[1]);
+	switch (type) {
+        case DHT_HASH_TYPE_DM_USER:
+                gf_log (this->name, GF_LOG_DEBUG, "found user-set layout");
+                layout->type = type;
+                /* Fall through. */
 	case DHT_HASH_TYPE_DM:
-		type = ntoh32 (disk_layout[1]);
 		break;
         default:
 		gf_log (this->name, GF_LOG_CRITICAL,
@@ -344,11 +334,12 @@ int
 dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                   int op_ret, int op_errno, dict_t *xattr)
 {
-        int      i     = 0;
-        int      ret   = -1;
-        int      err   = -1;
-        void    *disk_layout_raw = NULL;
-        int      disk_layout_len = 0;
+        int         i     = 0;
+        int         ret   = -1;
+        int         err   = -1;
+        void       *disk_layout_raw = NULL;
+        int         disk_layout_len = 0;
+        dht_conf_t *conf = this->private;
 
         if (op_ret != 0) {
                 err = op_errno;
@@ -369,12 +360,12 @@ dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
 
         if (xattr) {
                 /* during lookup and not mkdir */
-                ret = dict_get_ptr_and_len (xattr, "trusted.glusterfs.dht",
+                ret = dict_get_ptr_and_len (xattr, conf->xattr_name,
 					    &disk_layout_raw, &disk_layout_len);
         }
 
         if (ret != 0) {
-                layout->list[i].err = -1;
+                layout->list[i].err = 0;
                 gf_log (this->name, GF_LOG_TRACE,
                         "missing disk layout on %s. err = %d",
                         subvol->name, err);
@@ -421,6 +412,22 @@ dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
         layout->list[j].err    = err_swap;
 }
 
+void
+dht_layout_range_swap (dht_layout_t *layout, int i, int j)
+{
+        uint32_t  start_swap = 0;
+        uint32_t  stop_swap = 0;
+
+        start_swap  = layout->list[i].start;
+        stop_swap   = layout->list[i].stop;
+
+        layout->list[i].start  = layout->list[j].start;
+        layout->list[i].stop   = layout->list[j].stop;
+
+        layout->list[j].start  = start_swap;
+        layout->list[j].stop   = stop_swap;
+}
+
 int64_t
 dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
 {
@@ -428,17 +435,39 @@ dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
                         layout->list[j].xlator->name));
 }
 
+
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator)
+{
+        int i = 0;
+
+        for (i = 0; i < layout->cnt; i++) {
+                /* Check if xlator is already part of layout, and layout is
+                 * non-zero. */
+                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
+                        if (layout->list[i].start != layout->list[i].stop)
+                                return _gf_true;
+                        break;
+                }
+        }
+        return _gf_false;
+}
+
 int64_t
 dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
 {
         int64_t diff = 0;
 
-        if (layout->list[i].err || layout->list[j].err)
-                diff = layout->list[i].err - layout->list[j].err;
-        else
-                diff = (int64_t) layout->list[i].start
-                        - (int64_t) layout->list[j].start;
+        /* swap zero'ed out layouts to front, if needed */
+        if (!layout->list[j].start && !layout->list[j].stop) {
+                diff = (int64_t) layout->list[i].stop
+                       - (int64_t) layout->list[j].stop;
+                       goto out;
+        }
+        diff = (int64_t) layout->list[i].start
+                - (int64_t) layout->list[j].start;
 
+out:
         return diff;
 }
 
@@ -487,7 +516,8 @@ dht_layout_sort_volname (dht_layout_t *layout)
 int
 dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
                       uint32_t *holes_p, uint32_t *overlaps_p,
-                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p)
+                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                      uint32_t *no_space_p)
 {
         uint32_t    overlaps = 0;
         uint32_t    missing  = 0;
@@ -500,30 +530,51 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
         uint32_t    prev_stop = 0;
         uint32_t    last_stop = 0;
         char        is_virgin = 1;
-
-        /* TODO: explain what is happening */
-
+        uint32_t    no_space  = 0;
+
+        /* This funtion scans through the layout spread of a directory to
+           check if there are any anomalies. Prior to calling this function
+           the layout entries should be sorted in the ascending order.
+
+           If the layout entry has err != 0
+                then increment the corresponding anomaly.
+           else
+                if (start of the current layout entry > stop + 1 of previous
+                   non erroneous layout entry)
+                        then it indicates a hole in the layout
+                if (start of the current layout entry < stop + 1 of previous
+                    non erroneous layout entry)
+                         then it indicates an overlap in the layout
+        */
         last_stop = layout->list[0].start - 1;
         prev_stop = last_stop;
 
         for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err) {
-                        switch (layout->list[i].err) {
-                        case -1:
-                        case ENOENT:
-                                missing++;
-                                break;
-                        case ENOTCONN:
-                                down++;
-                                break;
-                        case ENOSPC:
-                                down++;
-                                break;
-                        default:
-                                misc++;
+                switch (layout->list[i].err) {
+                case -1:
+                case ENOENT:
+                case ESTALE:
+                        missing++;
+                        continue;
+                case ENOTCONN:
+                        down++;
+                        continue;
+                case ENOSPC:
+                        no_space++;
+                        continue;
+                case 0:
+                        /* if err == 0 and start == stop, then it is a non misc++;
+                         * participating subvolume(spread-cnt). Then, do not
+                         * check for anomalies. If start != stop, then treat it
+                         * as misc err */
+                        if (layout->list[i].start == layout->list[i].stop) {
+                                continue;
                         }
+                        break;
+                default:
+                        misc++;
                         continue;
-                }
+                 }
 
                 is_virgin = 0;
 
@@ -556,6 +607,9 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
         if (misc_p)
                 *misc_p = misc;
 
+        if (no_space_p)
+                *no_space_p = no_space;
+
         return ret;
 }
 
@@ -571,7 +625,6 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
         uint32_t     down = 0;
         uint32_t     misc = 0;
 
-
         ret = dht_layout_sort (layout);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
@@ -581,7 +634,7 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
 
         ret = dht_layout_anomalies (this, loc, layout,
                                     &holes, &overlaps,
-                                    &missing, &down, &misc);
+                                    &missing, &down, &misc, NULL);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "error while finding anomalies in %s -- not good news",
@@ -599,43 +652,56 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
                                 "found anomalies in %s. holes=%d overlaps=%d",
                                 loc->path, holes, overlaps);
                 }
-                ret = 1;
+                ret = -1;
         }
 
         for (i = 0; i < layout->cnt; i++) {
-                /* TODO During DHT selfheal rewrite (almost) find a better place to
-                 * detect this - probably in dht_layout_anomalies()
+                /* TODO During DHT selfheal rewrite (almost) find a better place
+                 * to detect this - probably in dht_layout_anomalies()
                  */
                 if (layout->list[i].err > 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "path=%s err=%s on subvol=%s",
-                                loc->path, strerror (layout->list[i].err),
-                                (layout->list[i].xlator ?
-                                 layout->list[i].xlator->name : "<>"));
-                        if (layout->list[i].err == ENOENT)
-                                ret = 1;
+                        gf_log_callingfn (this->name, GF_LOG_DEBUG,
+                                          "path=%s err=%s on subvol=%s",
+                                          loc->path,
+                                          strerror (layout->list[i].err),
+                                          (layout->list[i].xlator ?
+                                           layout->list[i].xlator->name
+                                           : "<>"));
+                        if ((layout->list[i].err == ENOENT) && (ret >= 0)) {
+                                ret++;
+                        }
                 }
         }
 
+
 out:
         return ret;
 }
 
+int
+dht_dir_has_layout (dict_t *xattr, char *name)
+{
+
+        void     *disk_layout_raw = NULL;
+
+        return dict_get_ptr (xattr, name, &disk_layout_raw);
+}
 
 int
 dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                          loc_t *loc, dict_t *xattr)
 {
-        int       idx = 0;
-        int       pos = -1;
-        int       ret = 0;
-        int       err = 0;
-        int       dict_ret = 0;
-        int32_t   disk_layout[4];
-        void     *disk_layout_raw = NULL;
-        int32_t   count = -1;
-        uint32_t  start_off = -1;
-        uint32_t  stop_off = -1;
+        int         idx = 0;
+        int         pos = -1;
+        int         ret = 0;
+        int         err = 0;
+        int         dict_ret = 0;
+        int32_t     disk_layout[4];
+        void       *disk_layout_raw = NULL;
+        int32_t     count = -1;
+        uint32_t    start_off = -1;
+        uint32_t    stop_off = -1;
+        dht_conf_t *conf = this->private;
 
 
         for (idx = 0; idx < layout->cnt; idx++) {
@@ -665,11 +731,11 @@ dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                 goto out;
         }
 
-        dict_ret = dict_get_ptr (xattr, "trusted.glusterfs.dht",
+        dict_ret = dict_get_ptr (xattr, conf->xattr_name,
                                  &disk_layout_raw);
 
         if (dict_ret < 0) {
-                if (err == 0) {
+                if (err == 0 && layout->list[pos].stop) {
                         gf_log (this->name, GF_LOG_INFO,
                                 "%s - disk layout missing", loc->path);
                         ret = -1;
@@ -730,7 +796,7 @@ dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
 
         LOCK (&conf->layout_lock);
         {
-                inode_ctx_put (inode, this, (uint64_t)(long)layout);
+                dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
 
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index d8ce49ddf..dbc9d0b3c 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -28,37 +19,106 @@
 #include "compat.h"
 #include "dht-common.h"
 
+int
+dht_linkfile_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno,
+                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                         struct iatt *postparent)
+{
+        char          is_linkfile   = 0;
+        dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
+        call_frame_t *prev          = NULL;
 
+        local = frame->local;
+        prev = cookie;
+        conf = this->private;
+
+        if (op_ret)
+                goto out;
+
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
+        if (!is_linkfile)
+                gf_log (this->name, GF_LOG_WARNING, "got non-linkfile %s:%s",
+                        prev->this->name, local->loc.path);
+out:
+        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
+                                      inode, stbuf, postparent, postparent,
+                                      xattr);
+        return 0;
+}
 
+#define is_equal(a, b) (a == b)
 int
 dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, inode_t *inode,
                          struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
+        xlator_t     *subvol = NULL;
+        call_frame_t *prev = NULL;
+        dict_t       *xattrs = NULL;
+        dht_conf_t   *conf = NULL;
+        int           ret = -1;
 
         local = frame->local;
 
+        if (!op_ret)
+                local->linked = _gf_true;
+
+        FRAME_SU_UNDO (frame, dht_local_t);
+
+        if (op_ret && (op_errno == EEXIST)) {
+                conf = this->private;
+                prev = cookie;
+                subvol = prev->this;
+                if (!subvol)
+                        goto out;
+                xattrs = dict_new ();
+                if (!xattrs)
+                        goto out;
+                ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set linkto key");
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_linkfile_lookup_cbk, subvol,
+                            subvol->fops->lookup, &local->loc, xattrs);
+                if (xattrs)
+                        dict_unref (xattrs);
+                return 0;
+        }
+out:
         local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, preparent, postparent);
+                                      inode, stbuf, preparent, postparent,
+                                      xdata);
+        if (xattrs)
+                dict_unref (xattrs);
         return 0;
 }
 
 
 int
 dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                     xlator_t *this,
                      xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
 {
         dht_local_t *local = NULL;
         dict_t      *dict = NULL;
         int          need_unref = 0;
         int          ret = 0;
+        dht_conf_t  *conf = this->private;
 
         local = frame->local;
         local->linkfile.linkfile_cbk = linkfile_cbk;
         local->linkfile.srcvol = tovol;
 
+        local->linked = _gf_false;
+
         dict = local->params;
         if (!dict) {
                 dict = dict_new ();
@@ -79,8 +139,7 @@ dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
                 gf_log ("dht-linkfile", GF_LOG_INFO,
                         "%s: internal-fop set failed", loc->path);
 
-        ret = dict_set_str (dict, "trusted.glusterfs.dht.linkto",
-                                   tovol->name);
+        ret = dict_set_str (dict, conf->link_xattr_name, tovol->name);
 
         if (ret < 0) {
                 gf_log (frame->this->name, GF_LOG_INFO,
@@ -89,9 +148,13 @@ dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
                 goto out;
         }
 
+        local->link_subvol = fromvol;
+        /* Always create as root:root. dht_linkfile_attr_heal fixes the
+         * ownsership */
+        FRAME_SU_DO (frame, dht_local_t);
         STACK_WIND (frame, dht_linkfile_create_cbk,
                     fromvol, fromvol->fops->mknod, loc,
-                    S_IFREG | DHT_LINKFILE_MODE, 0, dict);
+                    S_IFREG | DHT_LINKFILE_MODE, 0, 0, dict);
 
         if (need_unref && dict)
                 dict_unref (dict);
@@ -99,7 +162,7 @@ dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
         return 0;
 out:
         local->linkfile.linkfile_cbk (frame, NULL, frame->this, -1, ENOMEM,
-                                      loc->inode, NULL, NULL, NULL);
+                                      loc->inode, NULL, NULL, NULL, NULL);
 
         if (need_unref && dict)
                 dict_unref (dict);
@@ -111,7 +174,8 @@ out:
 int
 dht_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent)
+                         struct iatt *preparent, struct iatt *postparent,
+                         dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         call_frame_t  *prev = NULL;
@@ -155,7 +219,7 @@ dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
                     subvol, subvol->fops->unlink,
-                    &unlink_local->loc);
+                    &unlink_local->loc, 0, NULL);
 
         return 0;
 err:
@@ -180,7 +244,7 @@ dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
         if (!xattr)
                 goto out;
 
-        ret = dict_get_ptr (xattr, "trusted.glusterfs.dht.linkto", &volname);
+        ret = dict_get_ptr (xattr, conf->link_xattr_name, &volname);
 
         if ((-1 == ret) || !volname)
                 goto out;
@@ -195,3 +259,70 @@ dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
 out:
         return subvol;
 }
+
+int
+dht_linkfile_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, struct iatt *statpre,
+                          struct iatt *statpost, dict_t *xdata)
+{
+        dht_local_t *local = NULL;
+        loc_t *loc = NULL;
+
+        local = frame->local;
+        loc = &local->loc;
+
+        if (op_ret)
+                gf_log (this->name, GF_LOG_ERROR, "setattr of uid/gid on %s"
+                        " :<gfid:%s> failed (%s)",
+                        (loc->path? loc->path: "NULL"),
+                        uuid_utoa(local->gfid), strerror(op_errno));
+
+        DHT_STACK_DESTROY (frame);
+
+       return 0;
+}
+
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this)
+{
+        int     ret = -1;
+        call_frame_t *copy = NULL;
+        dht_local_t  *local = NULL;
+        dht_local_t  *copy_local = NULL;
+        xlator_t     *subvol = NULL;
+        struct iatt   stbuf = {0,};
+
+        local = frame->local;
+
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
+
+        if (local->stbuf.ia_type == IA_INVAL)
+                return 0;
+
+        uuid_copy (local->loc.gfid, local->stbuf.ia_gfid);
+
+        copy = copy_frame (frame);
+
+        if (!copy)
+                goto out;
+
+       copy_local = dht_local_init (copy, &local->loc, NULL, 0);
+
+       if (!copy_local)
+               goto out;
+
+        stbuf = local->stbuf;
+        subvol = local->link_subvol;
+
+        copy->local = copy_local;
+
+        FRAME_SU_DO (copy, dht_local_t);
+
+        STACK_WIND (copy, dht_linkfile_setattr_cbk, subvol,
+                    subvol->fops->setattr, &copy_local->loc,
+                    &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
index cc01e8f4b..e893eb48f 100644
--- a/xlators/cluster/dht/src/dht-mem-types.h
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -37,6 +28,8 @@ enum gf_dht_mem_types_ {
         gf_dht_mt_subvol_time,
         gf_dht_mt_loc_t,
         gf_defrag_info_mt,
+        gf_dht_mt_inode_ctx_t,
+        gf_dht_mt_ctx_stat_time_t,
         gf_dht_mt_end
 };
 #endif
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 0df20131a..4f78f5203 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -24,6 +15,9 @@
 #endif
 
 #include "dht-common.h"
+#include "xlator.h"
+#include <signal.h>
+#include <fnmatch.h>
 
 #define GF_DISK_SECTOR_SIZE             512
 #define DHT_REBALANCE_PID               4242 /* Change it if required */
@@ -64,7 +58,8 @@ dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
                                 if (ret < 0) {
                                         gf_log (THIS->name, GF_LOG_WARNING,
                                                 "failed to write (%s)",
-                                                strerror (errno));
+                                                strerror (-ret));
+                                        ret = -1;
                                         goto out;
                                 }
 
@@ -82,7 +77,8 @@ dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
                                 /* 'path' will be logged in calling function */
                                 gf_log (THIS->name, GF_LOG_WARNING,
                                         "failed to write (%s)",
-                                        strerror (errno));
+                                        strerror (-ret));
+                                ret = -1;
                                 goto out;
                         }
                 }
@@ -98,6 +94,41 @@ out:
 
 }
 
+/*
+   return values:
+   -1 : failure
+   -2 : success
+
+Hard link migration is carried out in three stages.
+
+(Say there are n hardlinks)
+Stage 1: Setting the new hashed subvol information on the 1st hardlink
+         encountered (linkto setxattr)
+
+Stage 2: Creating hardlinks on new hashed subvol for the 2nd to (n-1)th
+         hardlink
+
+Stage 3: Physical migration of the data file for nth hardlink
+
+Why to deem "-2" as success and not "0":
+
+   dht_migrate_file expects return value "0" from _is_file_migratable if
+the file has to be migrated.
+
+   _is_file_migratable returns zero only when it is called with the
+flag "GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS".
+
+   gf_defrag_handle_hardlink calls dht_migrate_file for physical migration
+of the data file with the flag "GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS"
+
+Hence, gf_defrag_handle_hardlink returning "0" for success will force
+"dht_migrate_file" to migrate each of the hardlink which is not intended.
+
+For each of the three stage mentioned above "-2" will be returned and will
+be converted to "0" in dht_migrate_file.
+
+*/
+
 int32_t
 gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
                            struct iatt *stbuf)
@@ -109,12 +140,16 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
         data_t                 *data            = NULL;
         struct iatt             iatt            = {0,};
         int32_t                 op_errno        = 0;
+        dht_conf_t             *conf            = NULL;
 
         GF_VALIDATE_OR_GOTO ("defrag", loc, out);
         GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
         GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);
         GF_VALIDATE_OR_GOTO ("defrag", this, out);
         GF_VALIDATE_OR_GOTO ("defrag", xattrs, out);
+        GF_VALIDATE_OR_GOTO ("defrag", this->private, out);
+
+        conf = this->private;
 
         if (uuid_is_null (loc->pargfid)) {
                 gf_log ("", GF_LOG_ERROR, "loc->pargfid is NULL for "
@@ -145,10 +180,10 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
         gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
                 "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
                 cached_subvol->name, hashed_subvol->name);
-        data = dict_get (xattrs, DHT_LINKFILE_KEY);
+        data = dict_get (xattrs, conf->link_xattr_name);
         /* set linkto on cached -> hashed if not present, else link it */
         if (!data) {
-                ret = dict_set_str (xattrs, DHT_LINKFILE_KEY,
+                ret = dict_set_str (xattrs, conf->link_xattr_name,
                                     hashed_subvol->name);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR, "Failed to set "
@@ -160,9 +195,11 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR, "Linkto setxattr "
                                 "failed %s -> %s (%s)", cached_subvol->name,
-                                loc->name, strerror (errno));
+                                loc->name, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
+                ret = -2;
                 goto out;
         } else {
                 linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattrs);
@@ -175,7 +212,8 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
 
                 ret = syncop_link (hashed_subvol, loc, loc);
                 if  (ret) {
-                        op_errno = errno;
+                        op_errno = -ret;
+                        ret = -1;
                         gf_log (this->name, GF_LOG_ERROR, "link of %s -> %s"
                                 " failed on  subvol %s (%s)", loc->name,
                                 uuid_utoa(loc->gfid),
@@ -187,7 +225,8 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
         ret = syncop_lookup (hashed_subvol, loc, NULL, &iatt, NULL, NULL);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed lookup %s on %s (%s)"
-                        , loc->name, hashed_subvol->name, strerror (errno));
+                        , loc->name, hashed_subvol->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -197,12 +236,19 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
                 if (ret)
                         goto out;
         }
-        ret = 0;
+        ret = -2;
 out:
         return ret;
 }
 
-
+/*
+     return values
+     0 : File will be migrated
+    -2 : File will not be migrated
+         (This is the return value from gf_defrag_handle_hardlink. Checkout
+         gf_defrag_handle_hardlink for description of "returning -2")
+    -1 : failure
+*/
 static inline int
 __is_file_migratable (xlator_t *this, loc_t *loc,
                       struct iatt *stbuf, dict_t *xattrs, int flags)
@@ -225,7 +271,12 @@ __is_file_migratable (xlator_t *this, loc_t *loc,
                 if (flags == GF_DHT_MIGRATE_HARDLINK) {
                         ret = gf_defrag_handle_hardlink (this, loc,
                                                          xattrs, stbuf);
-                        if (ret) {
+
+                   /*
+                   Returning zero will force the file to be remigrated.
+                   Checkout gf_defrag_handle_hardlink for more information.
+                   */
+                        if (ret && ret != -2) {
                                 gf_log (this->name, GF_LOG_WARNING,
                                         "%s: failed to migrate file with link",
                                         loc->path);
@@ -233,8 +284,8 @@ __is_file_migratable (xlator_t *this, loc_t *loc,
                 } else {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: file has hardlinks", loc->path);
+                        ret = -ENOTSUP;
                 }
-                ret = ENOTSUP;
                 goto out;
         }
 
@@ -246,14 +297,16 @@ out:
 
 static inline int
 __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
-                                 dict_t *dict, fd_t **dst_fd)
+                                 dict_t *dict, fd_t **dst_fd, dict_t *xattr)
 {
-        xlator_t *this = NULL;
-        int       ret  = -1;
-        fd_t     *fd   = NULL;
-        struct iatt new_stbuf = {0,};
+        xlator_t    *this = NULL;
+        int          ret  = -1;
+        fd_t        *fd   = NULL;
+        struct iatt  new_stbuf = {0,};
+        dht_conf_t  *conf = NULL;
 
         this = THIS;
+        conf = this->private;
 
         ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);
         if (ret) {
@@ -262,7 +315,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc
                 goto out;
         }
 
-        ret = dict_set_str (dict, DHT_LINKFILE_KEY, from->name);
+        ret = dict_set_str (dict, conf->link_xattr_name, from->name);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set gfid in dict for create", loc->path);
@@ -289,25 +342,47 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc
                         goto out;
                 }
         }
-        if ((ret == -1) && (errno != ENOENT)) {
+        if ((ret < 0) && (-ret != ENOENT)) {
                 /* File exists in destination, but not accessible */
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "%s: failed to lookup file (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
         /* Create the destination with LINKFILE mode, and linkto xattr,
            if the linkfile already exists, it will just open the file */
         ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
-                             dict);
+                             dict, &new_stbuf);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to create %s on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
+        ret = syncop_fsetxattr (to, fd, xattr, 0);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set xattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+
+        ret = syncop_ftruncate (to, fd, stbuf->ia_size);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "ftruncate failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+
+        ret = syncop_fsetattr (to, fd, stbuf,
+                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                NULL, NULL);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "chown failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+
         if (dst_fd)
                 *dst_fd = fd;
 
@@ -327,13 +402,17 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         int             ret        = -1;
         xlator_t       *this       = NULL;
 
+        uint64_t        src_statfs_blocks = 1;
+        uint64_t        dst_statfs_blocks = 1;
+
         this = THIS;
 
         ret = syncop_statfs (from, loc, &src_statfs);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to get statfs of %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -341,7 +420,8 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to get statfs of %s on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -350,22 +430,34 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         if (flag != GF_DHT_MIGRATE_DATA)
                 goto check_avail_space;
 
-        if (((dst_statfs.f_bavail *
-              dst_statfs.f_bsize) / GF_DISK_SECTOR_SIZE) <
-            (((src_statfs.f_bavail * src_statfs.f_bsize) /
-              GF_DISK_SECTOR_SIZE) - stbuf->ia_blocks)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "data movement attempted from node (%s) with"
-                        " higher disk space to a node (%s) with "
-                        "lesser disk space (%s)", from->name,
-                        to->name, loc->path);
-
-                /* this is not a 'failure', but we don't want to
-                   consider this as 'success' too :-/ */
-                ret = 1;
-                goto out;
+        /* Check:
+           During rebalance `migrate-data` - Destination subvol experiences
+           a `reduction` in 'blocks' of free space, at the same time source
+           subvol gains certain 'blocks' of free space. A valid check is
+           necessary here to avoid errorneous move to destination where
+           the space could be scantily available.
+         */
+        if (stbuf) {
+                dst_statfs_blocks = ((dst_statfs.f_bavail *
+                                      dst_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                src_statfs_blocks = ((src_statfs.f_bavail *
+                                      src_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                if ((dst_statfs_blocks - stbuf->ia_blocks) <
+                    (src_statfs_blocks + stbuf->ia_blocks)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "data movement attempted from node (%s) with"
+                                " higher disk space to a node (%s) with "
+                                "lesser disk space (%s)", from->name,
+                                to->name, loc->path);
+
+                        /* this is not a 'failure', but we don't want to
+                           consider this as 'success' too :-/ */
+                        ret = 1;
+                        goto out;
+                }
         }
-
 check_avail_space:
         if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
               GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
@@ -416,8 +508,7 @@ __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst
                 offset += ret;
                 total += ret;
 
-                if (vector)
-                        GF_FREE (vector);
+                GF_FREE (vector);
                 if (iobref)
                         iobref_unref (iobref);
                 iobref = NULL;
@@ -425,11 +516,12 @@ __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst
         }
         if (iobref)
                 iobref_unref (iobref);
-        if (vector)
-                GF_FREE (vector);
+        GF_FREE (vector);
 
         if (ret >= 0)
                 ret = 0;
+        else
+                ret = -1;
 
         return ret;
 }
@@ -444,8 +536,10 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         dict_t      *dict = NULL;
         xlator_t    *this = NULL;
         struct iatt  iatt = {0,};
+        dht_conf_t  *conf = NULL;
 
         this = THIS;
+        conf = this->private;
 
         fd = fd_create (loc->inode, DHT_REBALANCE_PID);
         if (!fd) {
@@ -456,10 +550,11 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         }
 
         ret = syncop_open (from, loc, O_RDWR, fd);
-        if (ret == -1) {
+        if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to open file %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -468,7 +563,7 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         if (!dict)
                 goto out;
 
-        ret = dict_set_str (dict, DHT_LINKFILE_KEY, to->name);
+        ret = dict_set_str (dict, conf->link_xattr_name, to->name);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to set xattr in dict for %s (linkto:%s)",
@@ -482,7 +577,8 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to set xattr on %s in %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -496,7 +592,8 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to set mode on %s in %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -521,12 +618,13 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
         dict_t      *dict     = NULL;
         char        *link     = NULL;
         struct iatt  stbuf    = {0,};
+        dht_conf_t  *conf     = this->private;
 
         dict = dict_new ();
         if (!dict)
                 goto out;
 
-        ret = dict_set_int32 (dict, DHT_LINKFILE_KEY, 256);
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set 'linkto' key in dict", loc->path);
@@ -535,19 +633,21 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
 
         /* check in the destination if the file is link file */
         ret = syncop_lookup (to, loc, dict, &stbuf, &rsp_dict, NULL);
-        if ((ret == -1) && (errno != ENOENT)) {
+        if ((ret < 0) && (-ret != ENOENT)) {
                 gf_log (this->name, GF_LOG_WARNING, "%s: lookup failed (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
         /* we no more require this key */
-        dict_del (dict, DHT_LINKFILE_KEY);
+        dict_del (dict, conf->link_xattr_name);
 
         /* file exists in target node, only if it is 'linkfile' its valid,
            otherwise, error out */
         if (!ret) {
-                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict)) {
+                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict,
+                                        conf->link_xattr_name)) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: file exists in destination", loc->path);
                         ret = -1;
@@ -559,7 +659,8 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: failed to delete the linkfile (%s)",
-                                loc->path, strerror (errno));
+                                loc->path, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
         }
@@ -579,15 +680,17 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: readlink on symlink failed (%s)",
-                                loc->path, strerror (errno));
+                                loc->path, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
 
-                ret = syncop_symlink (to, loc, link, dict);
+                ret = syncop_symlink (to, loc, link, dict, 0);
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: creating symlink failed (%s)",
-                                loc->path, strerror (errno));
+                                loc->path, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
 
@@ -597,18 +700,31 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
         ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
                                                       buf->ia_type),
                             makedev (ia_major (buf->ia_rdev),
-                                     ia_minor (buf->ia_rdev)), dict);
+                                     ia_minor (buf->ia_rdev)), dict, 0);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "%s: mknod failed (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
 done:
+        ret = syncop_setattr (to, loc, buf,
+                              (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
+                               GF_SET_ATTR_MODE), NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
+
         ret = syncop_unlink (from, loc);
-        if (ret)
+        if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "%s: unlink failed (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
+        }
 
 out:
         if (dict)
@@ -642,6 +758,7 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         dict_t         *xattr          = NULL;
         dict_t         *xattr_rsp      = NULL;
         int             file_has_holes = 0;
+        dht_conf_t     *conf           = this->private;
 
         gf_log (this->name, GF_LOG_INFO, "%s: attempting to move from %s to %s",
                 loc->path, from->name, to->name);
@@ -650,7 +767,7 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (!dict)
                 goto out;
 
-        ret = dict_set_int32 (dict, DHT_LINKFILE_KEY, 256);
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set 'linkto' key in dict", loc->path);
@@ -661,21 +778,24 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         ret = syncop_lookup (from, loc, dict, &stbuf, &xattr_rsp, NULL);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "%s: lookup failed on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
         /* we no more require this key */
-        dict_del (dict, DHT_LINKFILE_KEY);
+        dict_del (dict, conf->link_xattr_name);
 
         /* preserve source mode, so set the same to the destination */
         src_ia_prot = stbuf.ia_prot;
 
         /* Check if file can be migrated */
         ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag);
-        if (ret)
+        if (ret) {
+                if (ret == -2)
+                        ret = 0;
                 goto out;
-
+        }
         /* Take care of the special files */
         if (!IA_ISREG (stbuf.ia_type)) {
                 /* Special files */
@@ -683,9 +803,18 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
+        /* TODO: move all xattr related operations to fd based operations */
+        ret = syncop_listxattr (from, loc, &xattr);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get xattr from %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+        }
+
         /* create the destination, with required modes/xattr */
         ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
-                                               dict, &dst_fd);
+                                               dict, &dst_fd, xattr);
         if (ret)
                 goto out;
 
@@ -702,10 +831,12 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
+
         ret = syncop_fstat (from, src_fd, &stbuf);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "failed to lookup %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -720,38 +851,26 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 gf_log (this->name, GF_LOG_ERROR, "%s: failed to migrate data",
                         loc->path);
                 /* reset the destination back to 0 */
-                /* TODO: revert back to syncop_ftruncate once its fixed */
-                ret = syncop_truncate (to, loc, 0);
+                ret = syncop_ftruncate (to, dst_fd, 0);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "%s: failed to reset target size back to 0 (%s)",
-                                loc->path, strerror (errno));
+                                loc->path, strerror (-ret));
                 }
 
                 ret = -1;
                 goto out;
         }
 
-        /* TODO: move all xattr related operations to fd based operations */
-        ret = syncop_listxattr (from, loc, &xattr);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get xattr from %s (%s)",
-                        loc->path, from->name, strerror (errno));
-
-        ret = syncop_setxattr (to, loc, xattr, 0);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set xattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-
         /* TODO: Sync the locks */
 
-        ret = syncop_fsync (to, dst_fd);
-        if (ret)
+        ret = syncop_fsync (to, dst_fd, 0);
+        if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to fsync on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
 
 
         /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
@@ -761,7 +880,8 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 /* Failed to get the stat info */
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to fstat file %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -783,7 +903,9 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to perform setattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+                goto out;
         }
 
         /* Because 'futimes' is not portable */
@@ -793,7 +915,8 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to perform setattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
         }
 
         /* Make the source as a linkfile first before deleting it */
@@ -803,7 +926,28 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,             \
                         "%s: failed to perform setattr on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+       /* Free up the data blocks on the source node, as the whole
+           file is migrated */
+        ret = syncop_ftruncate (from, src_fd, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform truncate on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+        }
+
+        /* remove the 'linkto' xattr from the destination */
+        ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform removexattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
         }
 
         /* Do a stat and check the gfid before unlink */
@@ -811,7 +955,9 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to do a stat on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
         }
 
         if (uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0) {
@@ -820,33 +966,18 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: failed to perform unlink on %s (%s)",
-                                loc->path, from->name, strerror (errno));
+                                loc->path, from->name, strerror (-ret));
+                        ret = -1;
+                        goto out;
                 }
         }
 
-        /* Free up the data blocks on the source node, as the whole
-           file is migrated */
-        /* TODO: revert back to syncop_ftruncate once its fixed */
-        ret = syncop_truncate (from, loc, 0);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform truncate on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-        }
-
-        /* remove the 'linkto' xattr from the destination */
-        ret = syncop_fremovexattr (to, dst_fd, DHT_LINKFILE_KEY);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform removexattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-        }
-
         ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "%s: failed to lookup the file on subvolumes (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
         }
 
         gf_log (this->name, GF_LOG_INFO,
@@ -936,7 +1067,7 @@ rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
                 op_errno = EPERM;
         }
 
-        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno);
+        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL);
         return 0;
 }
 
@@ -944,25 +1075,21 @@ int
 dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
 {
         int         ret     = -1;
-        dht_conf_t *conf    = NULL;
-
-        conf = this->private;
 
-        ret = synctask_new (conf->env, rebalance_task,
+        ret = synctask_new (this->ctx->env, rebalance_task,
                             rebalance_task_completion,
                             frame, frame);
         return ret;
 }
 
 int
-gf_listener_stop (void)
+gf_listener_stop (xlator_t *this)
 {
         glusterfs_ctx_t  *ctx = NULL;
         cmd_args_t       *cmd_args = NULL;
         int              ret = 0;
-        xlator_t         *this = NULL;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = this->ctx;
         GF_ASSERT (ctx);
         cmd_args = &ctx->cmd_args;
         if (cmd_args->sock_file) {
@@ -973,7 +1100,6 @@ gf_listener_stop (void)
         }
 
         if (ret) {
-                this = THIS;
                 gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
                         "socket %s, error: %s", cmd_args->sock_file,
                         strerror (errno));
@@ -1014,10 +1140,10 @@ gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
 {
         /* if errno is not ENOSPC or ENOTCONN, we can still continue
            with rebalance process */
-        if ((errno != ENOSPC) || (errno != ENOTCONN))
+        if ((op_errno != ENOSPC) || (op_errno != ENOTCONN))
                 return 1;
 
-        if (errno == ENOTCONN) {
+        if (op_errno == ENOTCONN) {
                 /* Most probably mount point went missing (mostly due
                    to a brick down), say rebalance failure to user,
                    let him restart it if everything is fine */
@@ -1025,7 +1151,7 @@ gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
                 return -1;
         }
 
-        if (errno == ENOSPC) {
+        if (op_errno == ENOSPC) {
                 /* rebalance process itself failed, may be
                    remote brick went down, or write failed due to
                    disk full etc etc.. */
@@ -1036,6 +1162,31 @@ gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
         return 0;
 }
 
+static gf_boolean_t
+gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size)
+{
+        gf_defrag_pattern_list_t *trav = NULL;
+        gf_boolean_t               match = _gf_false;
+        gf_boolean_t               ret = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("dht", defrag, out);
+
+        trav = defrag->defrag_pattern;
+        while (trav) {
+                if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) {
+                        match = _gf_true;
+                        break;
+                }
+                trav = trav->next;
+        }
+
+        if ((match == _gf_true) && (size >= trav->size))
+                ret = _gf_true;
+
+ out:
+        return ret;
+}
+
 /* We do a depth first traversal of directories. But before we move into
  * subdirs, we complete the data migration of those directories whose layouts
  * have been fixed
@@ -1058,9 +1209,17 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         int32_t                  op_errno       = 0;
         char                    *uuid_str       = NULL;
         uuid_t                   node_uuid      = {0,};
-
-        gf_log (this->name, GF_LOG_INFO, "migate data called on %s",
+        struct timeval           dir_start      = {0,};
+        struct timeval           end            = {0,};
+        double                   elapsed        = {0,};
+        struct timeval           start          = {0,};
+        int32_t                  err            = 0;
+        int                      loglevel       = GF_LOG_TRACE;
+
+        gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
                 loc->path);
+        gettimeofday (&dir_start, NULL);
+
         fd = fd_create (loc->inode, defrag->pid);
         if (!fd) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
@@ -1071,21 +1230,29 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
                         loc->path);
+                ret = -1;
                 goto out;
         }
 
         INIT_LIST_HEAD (&entries.list);
 
         while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
-                &entries)) != 0)
-        {
-                if ((ret < 0) || (ret && (errno == ENOENT)))
-                        break;
+                                       &entries)) != 0) {
 
-                free_entries = _gf_true;
+                if (ret < 0) {
+
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s."
+                                " Aborting migrate-data",
+                                strerror(-ret));
+                        ret = -1;
+                        goto out;
+                }
 
                 if (list_empty (&entries.list))
                         break;
+
+                free_entries = _gf_true;
+
                 list_for_each_entry_safe (entry, tmp, &entries.list, list) {
                         if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
                                 ret = 1;
@@ -1102,7 +1269,15 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                 continue;
 
                         defrag->num_files_lookedup++;
-
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&start, NULL);
+                        }
+                        if (defrag->defrag_pattern &&
+                            (gf_defrag_pattern_match (defrag, entry->d_name,
+                                                      entry->d_stat.ia_size)
+                             == _gf_false)) {
+                                continue;
+                        }
                         loc_wipe (&entry_loc);
                         ret =dht_build_child_loc (this, &entry_loc, loc,
                                                   entry->d_name);
@@ -1137,6 +1312,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "%s"
                                         " lookup failed", entry_loc.path);
+                                ret = -1;
                                 continue;
                         }
 
@@ -1145,6 +1321,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if(ret < 0) {
                                 gf_log (this->name, GF_LOG_ERROR, "Failed to "
                                         "get node-uuid for %s", entry_loc.path);
+                                ret = -1;
                                 continue;
                         }
 
@@ -1154,6 +1331,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                 gf_log (this->name, GF_LOG_ERROR, "Failed to "
                                         "get node-uuid from dict for %s",
                                         entry_loc.path);
+                                ret = -1;
                                 continue;
                         }
 
@@ -1178,33 +1356,56 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
 
 
                         /* if distribute is present, it will honor this key.
-                         * -1 is returned if distribute is not present or file
-                         * doesn't have a link-file. If file has link-file, the
-                         * path of link-file will be the value, and also that
-                         * guarantees that file has to be mostly migrated */
+                         * -1, ENODATA is returned if distribute is not present
+                         * or file doesn't have a link-file. If file has
+                         * link-file, the path of link-file will be the value,
+                         * and also that guarantees that file has to be mostly
+                         * migrated */
 
                         ret = syncop_getxattr (this, &entry_loc, &dict,
                                                GF_XATTR_LINKINFO_KEY);
                         if (ret < 0) {
-                                gf_log (this->name, GF_LOG_TRACE, "getxattr "
-                                        "failed for %s", entry_loc.path);
+                                if (-ret != ENODATA) {
+                                        loglevel = GF_LOG_ERROR;
+                                        defrag->total_failures += 1;
+                                } else {
+                                        loglevel = GF_LOG_TRACE;
+                                }
+                                gf_log (this->name, loglevel, "%s: failed to "
+                                        "get "GF_XATTR_LINKINFO_KEY" key - %s",
+                                        entry_loc.path, strerror (-ret));
+                                ret = -1;
                                 continue;
                         }
 
                         ret = syncop_setxattr (this, &entry_loc, migrate_data,
                                                0);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR, "setxattr "
-                                        "failed for %s", entry_loc.path);
+                        if (ret) {
+                                err = op_errno;
+                                /* errno is overloaded. See
+                                 * rebalance_task_completion () */
+                                if (err != ENOSPC) {
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data skipped for %s"
+                                                " due to space constraints",
+                                                entry_loc.path);
+                                        defrag->skipped +=1;
+                                } else{
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "migrate-data failed for %s",
+                                                entry_loc.path);
+                                        defrag->total_failures +=1;
+                                }
+                        }
 
-                        if (ret == -1) {
-                                op_errno = errno;
+                        if (ret < 0) {
+                                op_errno = -ret;
                                 ret = gf_defrag_handle_migrate_error (op_errno,
                                                                       defrag);
 
                                 if (!ret)
                                         gf_log (this->name, GF_LOG_DEBUG,
-                                                "setxattr on %s failed: %s",
+                                                "migrate-data on %s failed: %s",
                                                 entry_loc.path,
                                                 strerror (op_errno));
                                 else if (ret == 1)
@@ -1219,13 +1420,27 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                 defrag->total_data += iatt.ia_size;
                         }
                         UNLOCK (&defrag->lock);
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&end, NULL);
+                                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
+                                          (end.tv_usec - start.tv_usec);
+                                gf_log (this->name, GF_LOG_INFO, "Migration of "
+                                        "file:%s size:%"PRIu64" bytes took %.2f"
+                                        "secs", entry_loc.path, iatt.ia_size,
+                                         elapsed/1e6);
+                        }
                 }
 
                 gf_dirent_free (&entries);
                 free_entries = _gf_false;
                 INIT_LIST_HEAD (&entries.list);
-
         }
+
+        gettimeofday (&end, NULL);
+        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
+                  (end.tv_usec - dir_start.tv_usec);
+        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
+                "%.2f secs", loc->path, elapsed/1e6);
         ret = 0;
 out:
         if (free_entries)
@@ -1242,7 +1457,6 @@ out:
 
 }
 
-
 int
 gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                   dict_t *fix_layout, dict_t *migrate_data)
@@ -1262,11 +1476,15 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Lookup failed on %s",
                         loc->path);
+                ret = -1;
                 goto out;
         }
 
-        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX)
-                gf_defrag_migrate_data (this, defrag, loc, migrate_data);
+        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                ret = gf_defrag_migrate_data (this, defrag, loc, migrate_data);
+                if (ret)
+                        goto out;
+        }
 
         gf_log (this->name, GF_LOG_TRACE, "fix layout called on %s", loc->path);
 
@@ -1289,12 +1507,19 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
                 &entries)) != 0)
         {
-                if ((ret < 0) || (ret && (errno == ENOENT)))
-                        break;
-                free_entries = _gf_true;
+
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s"
+                                ". Aborting fix-layout",strerror(-ret));
+                        ret = -1;
+                        goto out;
+                }
 
                 if (list_empty (&entries.list))
                         break;
+
+                free_entries = _gf_true;
+
                 list_for_each_entry_safe (entry, tmp, &entries.list, list) {
                         if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
                                 ret = 1;
@@ -1321,7 +1546,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
 
                         if (uuid_is_null (entry->d_stat.ia_gfid)) {
                                 gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        "gfid not present", loc->path,
+                                        " gfid not present", loc->path,
                                          entry->d_name);
                                 continue;
                         }
@@ -1331,7 +1556,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
                         if (uuid_is_null (loc->gfid)) {
                                 gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        "gfid not present", loc->path,
+                                        " gfid not present", loc->path,
                                          entry->d_name);
                                 continue;
                         }
@@ -1343,6 +1568,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "%s"
                                         " lookup failed", entry_loc.path);
+                                ret = -1;
                                 continue;
                         }
 
@@ -1353,6 +1579,8 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                         "failed for %s", entry_loc.path);
                                 defrag->defrag_status =
                                 GF_DEFRAG_STATUS_FAILED;
+                                defrag->total_failures ++;
+                                ret = -1;
                                 goto out;
                         }
                         ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
@@ -1361,6 +1589,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "Fix layout "
                                         "failed for %s", entry_loc.path);
+                                defrag->total_failures++;
                                 goto out;
                         }
 
@@ -1400,11 +1629,17 @@ gf_defrag_start_crawl (void *data)
         struct iatt              parent = {0,};
         dict_t                  *fix_layout = NULL;
         dict_t                  *migrate_data = NULL;
+        dict_t                  *status = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
 
         this = data;
         if (!this)
                 goto out;
 
+        ctx = this->ctx;
+        if (!ctx)
+                goto out;
+
         conf = this->private;
         if (!conf)
                 goto out;
@@ -1412,6 +1647,8 @@ gf_defrag_start_crawl (void *data)
         defrag = conf->defrag;
         if (!defrag)
                 goto out;
+
+        gettimeofday (&defrag->start_time, NULL);
         dht_build_root_inode (this, &defrag->root_inode);
         if (!defrag->root_inode)
                 goto out;
@@ -1424,6 +1661,7 @@ gf_defrag_start_crawl (void *data)
 
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "look up on / failed");
+                ret = -1;
                 goto out;
         }
 
@@ -1443,6 +1681,8 @@ gf_defrag_start_crawl (void *data)
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "fix layout on %s failed",
                         loc.path);
+                defrag->total_failures++;
+                ret = -1;
                 goto out;
         }
 
@@ -1464,17 +1704,30 @@ gf_defrag_start_crawl (void *data)
         }
         ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout,
                                     migrate_data);
+        if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
+            (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
+                defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+        }
+
+
 
 out:
         LOCK (&defrag->lock);
         {
-                gf_defrag_status_get (defrag, NULL);
+                status = dict_new ();
+                gf_defrag_status_get (defrag, status);
+                if (ctx->notify)
+                        ctx->notify (GF_EN_DEFRAG_STATUS, status);
+                if (status)
+                        dict_unref (status);
                 defrag->is_exiting = 1;
         }
         UNLOCK (&defrag->lock);
 
-        if (defrag)
+        if (defrag) {
                 GF_FREE (defrag);
+                conf->defrag = NULL;
+        }
 
         return ret;
 }
@@ -1483,9 +1736,8 @@ out:
 static int
 gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
 {
-        gf_listener_stop();
+        gf_listener_stop (sync_frame->this);
 
-        GF_FREE (data);
         STACK_DESTROY (sync_frame->root);
         kill (getpid(), SIGTERM);
         return 0;
@@ -1513,6 +1765,8 @@ gf_defrag_start (void *data)
         if (!frame)
                 goto out;
 
+        frame->root->pid = GF_CLIENT_PID_DEFRAG;
+
         defrag->pid = frame->root->pid;
 
         defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
@@ -1534,6 +1788,12 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         uint64_t files  = 0;
         uint64_t size   = 0;
         uint64_t lookup = 0;
+        uint64_t failures = 0;
+        uint64_t skipped = 0;
+        char     *status = "";
+        double   elapsed = 0;
+        struct timeval end = {0,};
+
 
         if (!defrag)
                 goto out;
@@ -1545,6 +1805,12 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         files  = defrag->total_files;
         size   = defrag->total_data;
         lookup = defrag->num_files_lookedup;
+        failures = defrag->total_failures;
+        skipped = defrag->skipped;
+
+        gettimeofday (&end, NULL);
+
+        elapsed = end.tv_sec - defrag->start_time.tv_sec;
 
         if (!dict)
                 goto log;
@@ -1564,13 +1830,53 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "failed to set lookedup file count");
 
+
         ret = dict_set_int32 (dict, "status", defrag->defrag_status);
         if (ret)
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "failed to set status");
+        if (elapsed) {
+                ret = dict_set_double (dict, "run-time", elapsed);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set run-time");
+        }
+
+        ret = dict_set_uint64 (dict, "failures", failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set failure count");
+
+        ret = dict_set_uint64 (dict, "skipped", skipped);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set skipped file count");
 log:
+        switch (defrag->defrag_status) {
+        case GF_DEFRAG_STATUS_NOT_STARTED:
+                status = "not started";
+                break;
+        case GF_DEFRAG_STATUS_STARTED:
+                status = "in progress";
+                break;
+        case GF_DEFRAG_STATUS_STOPPED:
+                status = "stopped";
+                break;
+        case GF_DEFRAG_STATUS_COMPLETE:
+                status = "completed";
+                break;
+        case GF_DEFRAG_STATUS_FAILED:
+                status = "failed";
+                break;
+        default:
+                break;
+        }
+
+        gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s. Time taken is %.2f "
+                "secs", status, elapsed);
         gf_log (THIS->name, GF_LOG_INFO, "Files migrated: %"PRIu64", size: %"
-                PRIu64", lookups: %"PRIu64, files, size, lookup);
+                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
+                "%"PRIu64, files, size, lookup, failures, skipped);
 
 
 out:
@@ -1578,7 +1884,8 @@ out:
 }
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output)
+gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
+                dict_t *output)
 {
         /* TODO: set a variable 'stop_defrag' here, it should be checked
            in defrag loop */
@@ -1589,11 +1896,13 @@ gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output)
                 goto out;
         }
 
-        defrag->defrag_status = GF_DEFRAG_STATUS_STOPPED;
+        gf_log ("", GF_LOG_INFO, "Received stop command on rebalance");
+        defrag->defrag_status = status;
 
-        gf_defrag_status_get (defrag, output);
+        if (output)
+                gf_defrag_status_get (defrag, output);
         ret = 0;
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index fa06de711..925538cc8 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /* TODO: link(oldpath, newpath) fails if newpath already exists. DHT should
@@ -35,7 +26,8 @@ int
 dht_rename_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = 0;
@@ -84,7 +76,7 @@ unwind:
                 DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                                   &local->stbuf, &local->preoldparent,
                                   &local->postoldparent,
-                                  &local->preparent, &local->postparent);
+                                  &local->preparent, &local->postparent, xdata);
         }
 
         return 0;
@@ -97,7 +89,7 @@ dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            struct iatt *preoldparent,
                            struct iatt *postoldparent,
                            struct iatt *prenewparent,
-                           struct iatt *postnewparent)
+                           struct iatt *postnewparent, dict_t *xdata)
 {
         dht_conf_t   *conf = NULL;
         dht_local_t  *local = NULL;
@@ -147,7 +139,7 @@ dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STACK_WIND (frame, dht_rename_dir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->rename,
-                            &local->loc, &local->loc2);
+                            &local->loc, &local->loc2, NULL);
                 if (!--call_cnt)
                         break;
         }
@@ -164,7 +156,7 @@ unwind:
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                           &local->stbuf, &local->preoldparent,
                           &local->postoldparent,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
 
         return 0;
 }
@@ -185,19 +177,20 @@ dht_rename_dir_do (call_frame_t *frame, xlator_t *this)
         STACK_WIND (frame, dht_rename_hashed_dir_cbk,
                     local->dst_hashed,
                     local->dst_hashed->fops->rename,
-                    &local->loc, &local->loc2);
+                    &local->loc, &local->loc2, NULL);
         return 0;
 
 err:
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL);
         return 0;
 }
 
 
 int
 dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries)
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
@@ -226,7 +219,7 @@ dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, fd_t *fd)
+                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         int           this_call_cnt = -1;
@@ -246,7 +239,7 @@ dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         STACK_WIND (frame, dht_rename_readdir_cbk,
                     prev->this, prev->this->fops->readdir,
-                    local->fd, 4096, 0);
+                    local->fd, 4096, 0, NULL);
 
         return 0;
 
@@ -302,22 +295,70 @@ dht_rename_dir (call_frame_t *frame, xlator_t *this)
                 STACK_WIND (frame, dht_rename_opendir_cbk,
                             conf->subvolumes[i],
                             conf->subvolumes[i]->fops->opendir,
-                            &local->loc2, local->fd);
+                            &local->loc2, local->fd, NULL);
         }
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL, NULL);
         return 0;
 }
+#define DHT_MARK_FOP_INTERNAL(xattr) do {                                      \
+                int tmp = -1;                                                  \
+                if (!xattr) {                                                  \
+                        xattr = dict_new ();                                   \
+                        if (!xattr)                                            \
+                                break;                                         \
+                }                                                              \
+                tmp = dict_set_str (xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); \
+                if (tmp) {                                                     \
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set"      \
+                                " internal dict key for %s", local->loc.path); \
+                }                                                              \
+        }while (0)
+
+#define DHT_MARKER_DONT_ACCOUNT(xattr) do {                             \
+                int tmp = -1;                                                  \
+                if (!xattr) {                                                  \
+                        xattr = dict_new ();                                   \
+                        if (!xattr)                                            \
+                                break;                                         \
+                }                                                              \
+                tmp = dict_set_str (xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY,  \
+                                    "yes");                                    \
+                if (tmp) {                                                     \
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set"      \
+                                " marker dont account key for %s", local->loc.path); \
+                }                                                              \
+        }while (0)
+
+int
+dht_rename_done (call_frame_t *frame, xlator_t *this)
+{
+        dht_local_t     *local = NULL;
+
+        local = frame->local;
+
+        if (local->linked == _gf_true) {
+                local->linked = _gf_false;
+                dht_linkfile_attr_heal (frame, this);
+        }
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                          &local->stbuf, &local->preoldparent,
+                          &local->postoldparent, &local->preparent,
+                          &local->postparent, NULL);
 
+        return 0;
+}
 
 int
 dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                       struct iatt *postparent)
+                       struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -346,11 +387,7 @@ dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         WIPE (&local->postparent);
 
         if (is_last_call (this_call_cnt)) {
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, &local->preoldparent,
-                                  &local->postoldparent, &local->preparent,
-                                  &local->postparent);
+                dht_rename_done (frame, this);
         }
 
 out:
@@ -368,7 +405,7 @@ dht_rename_cleanup (call_frame_t *frame)
         xlator_t    *dst_hashed = NULL;
         xlator_t    *dst_cached = NULL;
         int          call_cnt = 0;
-
+        dict_t      *xattr      = NULL;
 
         local = frame->local;
         this  = frame->this;
@@ -392,24 +429,53 @@ dht_rename_cleanup (call_frame_t *frame)
         if (!call_cnt)
                 goto nolinks;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
+
         if (dst_hashed != src_hashed && dst_hashed != src_cached) {
+                dict_t *xattr_new = NULL;
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "unlinking linkfile %s @ %s => %s",
                         local->loc.path, dst_hashed->name, src_cached->name);
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
+
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             dst_hashed, dst_hashed->fops->unlink,
-                            &local->loc);
+                            &local->loc, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
         }
 
         if (src_cached != dst_hashed) {
+                dict_t *xattr_new = NULL;
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "unlinking link %s => %s (%s)", local->loc.path,
                         local->loc2.path, src_cached->name);
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
+
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_cached, src_cached->fops->unlink,
-                            &local->loc2);
+                            &local->loc2, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
         }
 
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 
 nolinks:
@@ -422,7 +488,7 @@ nolinks:
         DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                           &local->stbuf, &local->preoldparent,
                           &local->postoldparent, &local->preparent,
-                          &local->postparent);
+                          &local->postparent, NULL);
 
         return 0;
 }
@@ -430,9 +496,10 @@ nolinks:
 
 int
 dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                             int32_t op_ret, int32_t op_errno,
+                             inode_t *inode, struct iatt *stbuf,
+                             struct iatt *preparent, struct iatt *postparent,
+                             dict_t *xdata)
 {
         call_frame_t *prev = NULL;
         dht_local_t  *local = NULL;
@@ -446,6 +513,10 @@ dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->loc.path, prev->this->name, strerror (op_errno));
         }
 
+        if (local->linked == _gf_true) {
+                local->linked = _gf_false;
+                dht_linkfile_attr_heal (frame, this);
+        }
         DHT_STACK_DESTROY (frame);
 
         return 0;
@@ -456,7 +527,8 @@ int
 dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
                 struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent)
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -467,6 +539,7 @@ dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         xlator_t     *rename_subvol = NULL;
         call_frame_t *link_frame = NULL;
         dht_local_t *link_local = NULL;
+        dict_t       *xattr     = NULL;
 
         local = frame->local;
         prev = cookie;
@@ -476,6 +549,8 @@ dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         dst_hashed = local->dst_hashed;
         dst_cached = local->dst_cached;
 
+        if (local->linked == _gf_true)
+                FRAME_SU_UNDO (frame, dht_local_t);
         if (op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: rename on %s failed (%s)", local->loc.path,
@@ -505,15 +580,25 @@ dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 uuid_copy (link_local->gfid, local->loc.inode->gfid);
 
                 dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
-                                     src_cached, dst_hashed, &link_local->loc);
+                                     this, src_cached, dst_hashed,
+                                     &link_local->loc);
         }
 
 err:
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent, prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent, prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent, prev->this);
+        /* Merge attrs only from src_cached. In case there of src_cached !=
+         * dst_hashed, this ignores linkfile attrs. */
+        if (prev->this == src_cached) {
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->preoldparent, preoldparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->postoldparent, postoldparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->preparent, prenewparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->postparent, postnewparent,
+                                prev->this);
+        }
+
 
         /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
          *       is called. since rename has already happened on rename_subvol,
@@ -538,24 +623,47 @@ err:
         if (local->call_cnt == 0)
                 goto unwind;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
+
         if (src_cached != dst_hashed && src_cached != dst_cached) {
+                dict_t *xattr_new = NULL;
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "deleting old src datafile %s @ %s",
                         local->loc.path, src_cached->name);
 
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
+
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_cached, src_cached->fops->unlink,
-                            &local->loc);
+                            &local->loc, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
         }
 
         if (src_hashed != rename_subvol && src_hashed != src_cached) {
+                dict_t *xattr_new = NULL;
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "deleting old src linkfile %s @ %s",
                         local->loc.path, src_hashed->name);
 
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_hashed, src_hashed->fops->unlink,
-                            &local->loc);
+                            &local->loc, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
         }
 
         if (dst_cached
@@ -567,8 +675,10 @@ err:
 
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             dst_cached, dst_cached->fops->unlink,
-                            &local->loc2);
+                            &local->loc2, 0, xattr);
         }
+        if (xattr)
+                dict_unref (xattr);
         return 0;
 
 unwind:
@@ -576,16 +686,16 @@ unwind:
         WIPE (&local->postoldparent);
         WIPE (&local->preparent);
         WIPE (&local->postparent);
+        if (xattr)
+                dict_unref (xattr);
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent, &local->preparent,
-                          &local->postparent);
+        dht_rename_done (frame, this);
 
         return 0;
 
 cleanup:
+        if (xattr)
+                dict_unref (xattr);
         dht_rename_cleanup (frame);
 
         return 0;
@@ -595,12 +705,13 @@ cleanup:
 int
 dht_do_rename (call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *dst_hashed = NULL;
-        xlator_t    *src_cached = NULL;
-        xlator_t    *dst_cached = NULL;
-        xlator_t    *this = NULL;
+        dht_local_t *local         = NULL;
+        xlator_t    *dst_hashed    = NULL;
+        xlator_t    *src_cached    = NULL;
+        xlator_t    *dst_cached    = NULL;
+        xlator_t    *this          = NULL;
         xlator_t    *rename_subvol = NULL;
+        dict_t      *dict          = NULL;
 
 
         local = frame->local;
@@ -615,13 +726,19 @@ dht_do_rename (call_frame_t *frame)
         else
                 rename_subvol = dst_hashed;
 
+        if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
+                DHT_MARKER_DONT_ACCOUNT(dict);
+        }
+
         gf_log (this->name, GF_LOG_TRACE,
                 "renaming %s => %s (%s)",
                 local->loc.path, local->loc2.path, rename_subvol->name);
 
+        if (local->linked == _gf_true)
+                FRAME_SU_DO (frame, dht_local_t);
         STACK_WIND (frame, dht_rename_cbk,
                     rename_subvol, rename_subvol->fops->rename,
-                    &local->loc, &local->loc2);
+                    &local->loc, &local->loc2, dict);
 
         return 0;
 }
@@ -631,7 +748,8 @@ int
 dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -648,6 +766,9 @@ dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->op_ret   = -1;
                 if (op_errno != ENOENT)
                         local->op_errno = op_errno;
+        } else if (local->src_cached == prev->this) {
+                /* merge of attr returned only from linkfile creation */
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
         }
 
         this_call_cnt = dht_frame_return (frame);
@@ -670,7 +791,8 @@ cleanup:
 int
 dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno,
-                             struct iatt *preparent, struct iatt *postparent)
+                             struct iatt *preparent, struct iatt *postparent,
+                             dict_t *xdata)
 {
 	dht_local_t  *local = NULL;
 	call_frame_t *prev = NULL;
@@ -712,6 +834,7 @@ dht_rename_create_links (call_frame_t *frame)
         xlator_t    *dst_hashed = NULL;
         xlator_t    *dst_cached = NULL;
         int          call_cnt = 0;
+        dict_t      *xattr = NULL;
 
 
         local = frame->local;
@@ -722,18 +845,27 @@ dht_rename_create_links (call_frame_t *frame)
         dst_hashed = local->dst_hashed;
         dst_cached = local->dst_cached;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
 
         if (src_cached == dst_cached) {
+                dict_t *xattr_new = NULL;
+
                 if (dst_hashed == dst_cached)
                         goto nolinks;
 
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
 		gf_log (this->name, GF_LOG_TRACE,
 			"unlinking dst linkfile %s @ %s",
 			local->loc2.path, dst_hashed->name);
 
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+
 		STACK_WIND (frame, dht_rename_unlink_links_cbk,
 			    dst_hashed, dst_hashed->fops->unlink,
-			    &local->loc2);
+			    &local->loc2, 0, xattr_new);
+
+                dict_unref (xattr_new);
                 return 0;
         }
 
@@ -750,17 +882,28 @@ dht_rename_create_links (call_frame_t *frame)
                         "linkfile %s @ %s => %s",
                         local->loc.path, dst_hashed->name, src_cached->name);
                 memcpy (local->gfid, local->loc.inode->gfid, 16);
-		dht_linkfile_create (frame, dht_rename_links_cbk,
+		dht_linkfile_create (frame, dht_rename_links_cbk, this,
 				     src_cached, dst_hashed, &local->loc);
 	}
 
 	if (src_cached != dst_hashed) {
+                dict_t *xattr_new = NULL;
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
 		gf_log (this->name, GF_LOG_TRACE,
 			"link %s => %s (%s)", local->loc.path,
 			local->loc2.path, src_cached->name);
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
+
 		STACK_WIND (frame, dht_rename_links_cbk,
 			    src_cached, src_cached->fops->link,
-			    &local->loc, &local->loc2);
+			    &local->loc, &local->loc2, xattr_new);
+
+                dict_unref (xattr_new);
 	}
 
 nolinks:
@@ -768,6 +911,8 @@ nolinks:
                 /* skip to next step */
                 dht_do_rename (frame);
         }
+        if (xattr)
+                dict_unref (xattr);
 
         return 0;
 }
@@ -775,7 +920,7 @@ nolinks:
 
 int
 dht_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc)
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         xlator_t    *src_cached = NULL;
         xlator_t    *src_hashed = NULL;
@@ -857,7 +1002,8 @@ dht_rename (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL, NULL);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 514eefa35..0e6527544 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -26,7 +17,7 @@
 #include "glusterfs.h"
 #include "xlator.h"
 #include "dht-common.h"
-
+#include "glusterfs-acl.h"
 
 #define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,cnt,path)    do {       \
                 layout->list[i].start = srt;                            \
@@ -38,43 +29,40 @@
                         layout->list[i].xlator->name, path);            \
         } while (0)
 
+#define DHT_RESET_LAYOUT_RANGE(layout)    do {                          \
+                int cnt = 0;                                            \
+                for (cnt = 0; cnt < layout->cnt; cnt++ ) {              \
+                        layout->list[cnt].start = 0;                    \
+                        layout->list[cnt].stop  = 0;                    \
+                }                                                       \
+        } while (0)
 
-static inline uint32_t
-dht_find_overlap (int idx, int cnk_idx, uint32_t start, uint32_t stop,
-                  uint32_t chunk_size)
+static uint32_t
+dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
 {
-        uint32_t overlap = 0;
-        uint32_t chunk_begin = 0;
+	if (o >= old->cnt || n >= new->cnt)
+		return 0;
 
-        chunk_begin = cnk_idx * chunk_size;
+	if (old->list[o].err > 0 || new->list[n].err > 0)
+		return 0;
 
-        /* There is no chance of overlap */
-        if ((chunk_begin > stop) ||
-            ((chunk_begin + chunk_size) < start))
-                goto out;
-
-        if ((chunk_begin <= start) &&
-            ((chunk_begin + chunk_size) <= stop)) {
-                overlap = ((chunk_begin + chunk_size) - start);
-                goto out;
+        if (old->list[o].start == old->list[o].stop) {
+                return 0;
         }
 
-        if ((chunk_begin <= start) &&
-            ((chunk_begin + chunk_size) >= stop)) {
-                overlap = (stop - start);
-                goto out;
+        if (new->list[n].start == new->list[n].stop) {
+                return 0;
         }
 
-        if ((chunk_begin < stop) &&
-            ((chunk_begin + chunk_size) >= stop)) {
-                overlap = (stop - chunk_begin);
-                goto out;
-        }
+	if ((old->list[o].start > new->list[n].stop) ||
+	    (old->list[o].stop < new->list[n].start))
+		return 0;
 
-out:
-        return overlap;
+	return min (old->list[o].stop, new->list[n].stop) -
+	        max (old->list[o].start, new->list[n].start) + 1;
 }
 
+
 int
 dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 {
@@ -82,7 +70,7 @@ dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 
         local = frame->local;
         local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
-                                 local->op_errno);
+                                 local->op_errno, NULL);
 
         return 0;
 }
@@ -90,7 +78,7 @@ dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 
 int
 dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno)
+                            int op_ret, int op_errno, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
@@ -129,7 +117,8 @@ dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-                                  dht_layout_t *layout, int i)
+                                  dht_layout_t *layout, int i,
+                                  xlator_t *req_subvol)
 {
         xlator_t          *subvol = NULL;
         dict_t            *xattr = NULL;
@@ -137,12 +126,24 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
         xlator_t          *this = NULL;
         int32_t           *disk_layout = NULL;
         dht_local_t       *local = NULL;
-
+        dht_conf_t        *conf = NULL;
+        data_t            *data = NULL;
 
         local = frame->local;
-        subvol = layout->list[i].xlator;
+        if (req_subvol)
+                subvol = req_subvol;
+        else
+                subvol = layout->list[i].xlator;
         this = frame->this;
 
+        GF_VALIDATE_OR_GOTO ("", this, err);
+        GF_VALIDATE_OR_GOTO (this->name, layout, err);
+        GF_VALIDATE_OR_GOTO (this->name, local, err);
+        GF_VALIDATE_OR_GOTO (this->name, subvol, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
+
         xattr = get_new_dict ();
         if (!xattr) {
                 goto err;
@@ -156,8 +157,7 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
                 goto err;
         }
 
-        ret = dict_set_bin (xattr, "trusted.glusterfs.dht",
-                            disk_layout, 4 * 4);
+        ret = dict_set_bin (xattr, conf->xattr_name, disk_layout, 4 * 4);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: (subvol %s) failed to set xattr dictionary",
@@ -172,13 +172,22 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
                 layout->type, subvol->name, loc->path);
 
         dict_ref (xattr);
-
+        if (local->xattr) {
+                data = dict_get (local->xattr, QUOTA_LIMIT_KEY);
+                if (data) {
+                        ret = dict_add (xattr, QUOTA_LIMIT_KEY, data);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "set quota limit key on %s",loc->path);
+                        }
+                }
+        }
         if (!uuid_is_null (local->gfid))
                 uuid_copy (loc->gfid, local->gfid);
 
         STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
                     subvol, subvol->fops->setxattr,
-                    loc, xattr, 0);
+                    loc, xattr, 0, NULL);
 
         dict_unref (xattr);
 
@@ -188,11 +197,10 @@ err:
         if (xattr)
                 dict_destroy (xattr);
 
-        if (disk_layout)
-                GF_FREE (disk_layout);
+        GF_FREE (disk_layout);
 
         dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
-                                    -1, ENOMEM);
+                                    -1, ENOMEM, NULL);
         return 0;
 }
 
@@ -203,21 +211,42 @@ dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
         int          i = 0;
         int          count = 0;
         xlator_t    *this = NULL;
+        dht_conf_t  *conf = NULL;
+        dht_layout_t *dummy = NULL;
 
         local = frame->local;
         this = frame->this;
+        conf = this->private;
 
         gf_log (this->name, GF_LOG_DEBUG,
                 "writing the new range for all subvolumes");
 
-        local->call_cnt = count = layout->cnt;
+        local->call_cnt = count = conf->subvolume_cnt;
 
         for (i = 0; i < layout->cnt; i++) {
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
 
                 if (--count == 0)
-                        break;
+                        goto out;
         }
+        /* if we are here, subvolcount > layout_count. subvols-per-directory
+         * option might be set here. We need to clear out layout from the
+         * non-participating subvolumes, else it will result in overlaps */
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                        if (--count == 0)
+                                break;
+                }
+        }
+
+        dht_layout_unref (this, dummy);
+out:
         return 0;
 }
 
@@ -228,9 +257,12 @@ dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
         int          missing_xattr = 0;
         int          i = 0;
         xlator_t    *this = NULL;
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *dummy = NULL;
 
         local = frame->local;
         this = frame->this;
+        conf = this->private;
 
         for (i = 0; i < layout->cnt; i++) {
                 if (layout->list[i].err != -1 || !layout->list[i].stop) {
@@ -243,7 +275,14 @@ dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
                 }
                 missing_xattr++;
         }
-
+        /* Also account for subvolumes with no-layout. Used for zero'ing out
+         * the layouts and for setting quota key's if present */
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        missing_xattr++;
+                }
+        }
         gf_log (this->name, GF_LOG_TRACE,
                 "%d subvolumes missing xattr for %s",
                 missing_xattr, loc->path);
@@ -254,23 +293,36 @@ dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
         }
 
         local->call_cnt = missing_xattr;
-
         for (i = 0; i < layout->cnt; i++) {
                 if (layout->list[i].err != -1 || !layout->list[i].stop)
                         continue;
 
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
 
                 if (--missing_xattr == 0)
                         break;
         }
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                        missing_xattr--;
+                }
+        }
+
+        dht_layout_unref (this, dummy);
+out:
         return 0;
 }
 
 int
 dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                               int op_ret, int op_errno, struct iatt *statpre,
-                              struct iatt *statpost)
+                              struct iatt *statpost, dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         dht_layout_t  *layout = NULL;
@@ -324,7 +376,7 @@ dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
                         STACK_WIND (frame, dht_selfheal_dir_setattr_cbk,
                                     layout->list[i].xlator,
                                     layout->list[i].xlator->fops->setattr,
-                                    loc, stbuf, valid);
+                                    loc, stbuf, valid, NULL);
                 }
         }
 
@@ -335,7 +387,8 @@ int
 dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             int op_ret, int op_errno,
                             inode_t *inode, struct iatt *stbuf,
-                            struct iatt *preparent, struct iatt *postparent)
+                            struct iatt *preparent, struct iatt *postparent,
+                            dict_t *xdata)
 {
         dht_local_t   *local = NULL;
         dht_layout_t  *layout = NULL;
@@ -381,6 +434,46 @@ out:
         return 0;
 }
 
+void
+dht_selfheal_dir_mkdir_setacl (dict_t *xattr, dict_t *dict)
+{
+        data_t          *acl_default = NULL;
+        data_t          *acl_access = NULL;
+        xlator_t        *this = NULL;
+        int     ret = -1;
+
+        GF_ASSERT (xattr);
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        acl_default = dict_get (xattr, POSIX_ACL_DEFAULT_XATTR);
+
+        if (!acl_default) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ACL_DEFAULT xattr not present");
+                goto cont;
+        }
+        ret = dict_set (dict, POSIX_ACL_DEFAULT_XATTR, acl_default);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not set ACL_DEFAULT xattr");
+cont:
+        acl_access = dict_get (xattr, POSIX_ACL_ACCESS_XATTR);
+        if (!acl_access) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ACL_ACCESS xattr not present");
+                goto out;
+        }
+        ret = dict_set (dict, POSIX_ACL_ACCESS_XATTR, acl_access);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not set ACL_ACCESS xattr");
+
+out:
+        return;
+}
 
 int
 dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
@@ -420,10 +513,13 @@ dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
                 /* Send the dictionary from higher layers directly */
                 dict = dict_ref (local->params);
         }
+        /* Set acls */
+        if (local->xattr && dict)
+                dht_selfheal_dir_mkdir_setacl (local->xattr, dict);
 
         if (!dict)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "dict is NULL, need to make sure gfid's are same");
+                        "dict is NULL, need to make sure gfids are same");
 
         for (i = 0; i < layout->cnt; i++) {
                 if (layout->list[i].err == ENOENT || force) {
@@ -437,7 +533,7 @@ dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
                                     loc,
                                     st_mode_from_ia (local->stbuf.ia_prot,
                                                      local->stbuf.ia_type),
-                                    dict);
+                                    0, dict);
                 }
         }
 
@@ -456,7 +552,7 @@ dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
         uint32_t      hashval = 0;
         int           ret = 0;
 
-        ret = dht_hash_compute (layout->type, loc->path, &hashval);
+        ret = dht_hash_compute (this, layout->type, loc->path, &hashval);
         if (ret == 0) {
                 start = (hashval % layout->cnt);
         }
@@ -479,7 +575,7 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
                 for (j = 0; j < conf->subvolume_cnt; j++) {
                         if (conf->decommissioned_bricks[j] &&
                             conf->decommissioned_bricks[j] == layout->list[i].xlator) {
-                                layout->list[i].err = -EINVAL;
+                                layout->list[i].err = EINVAL;
                                 break;
                         }
                 }
@@ -487,9 +583,33 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
 
         for (i = 0; i < layout->cnt; i++) {
                 err = layout->list[i].err;
-                if (err == -1 || err == 0) {
-                        layout->list[i].err = -1;
+                if (err == -1 || err == 0 || err == ENOENT) {
+			/* Setting list[i].err = -1 is an indication for
+			   dht_selfheal_layout_new_directory() to assign
+			   a range. We set it to -1 based on any one of
+			   the three criteria:
+
+			   - err == -1 already, which means directory
+			     existed but layout was not set on it.
+
+			   - err == 0, which means directory exists and
+			     has an old layout piece which will be
+			     overwritten now.
+
+			   - err == ENOENT, which means directory does
+			     not exist (possibly racing with mkdir or
+			     finishing half done mkdir). The missing
+			     directory will be attempted to be recreated.
+
+			     It is important to note that it is safe
+			     to race with mkdir() as self-heal and
+			     mkdir are idempotent operations. Both will
+			     strive to set the directory and layouts to
+			     the same final state.
+			*/
                         count++;
+			if (!err)
+				layout->list[i].err = -1;
                 }
         }
 
@@ -504,49 +624,126 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
                 }
         }
 
-        count = ((layout->spread_cnt) ? layout->spread_cnt :
-                 ((count) ? count : 1));
+        /* if layout->spread_cnt is set, check if it is <= available
+         * subvolumes (down brick and decommissioned bricks are considered
+         * un-availbale). Else return count (available up bricks) */
+        count = ((layout->spread_cnt &&
+                 (layout->spread_cnt <= count)) ?
+                 layout->spread_cnt : ((count) ? count : 1));
 
         return count;
 }
 
 
+void dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
+					dht_layout_t *new_layout);
+
+void dht_layout_entry_swap (dht_layout_t *layout, int i, int j);
+void dht_layout_range_swap (dht_layout_t *layout, int i, int j);
+
+/*
+ * It's a bit icky using local variables in a macro, but it makes the rest
+ * of the code a lot clearer.
+ */
+#define OV_ENTRY(x,y)      table[x*new->cnt+y]
+
+void
+dht_selfheal_layout_maximize_overlap (call_frame_t *frame, loc_t *loc,
+				      dht_layout_t *new, dht_layout_t *old)
+{
+	int           i            = 0;
+	int           j            = 0;
+	uint32_t      curr_overlap = 0;
+	uint32_t      max_overlap  = 0;
+	int           max_overlap_idx = -1;
+	uint32_t      overlap      = 0;
+        uint32_t     *table = NULL;
+
+	dht_layout_sort_volname (old);
+	/* Now both old_layout->list[] and new_layout->list[]
+	   are match the same xlators/subvolumes. i.e,
+	   old_layout->[i] and new_layout->[i] are referring
+	   to the same subvolumes
+	*/
+
+        /* Build a table of overlaps between new[i] and old[j]. */
+        table = alloca(sizeof(overlap)*old->cnt*new->cnt);
+        if (!table) {
+                return;
+        }
+        memset(table,0,sizeof(overlap)*old->cnt*new->cnt);
+        for (i = 0; i < new->cnt; ++i) {
+                for (j = 0; j < old->cnt; ++j) {
+                        OV_ENTRY(i,j) = dht_overlap_calc(old,j,new,i);
+                }
+        }
+
+	for (i = 0; i < new->cnt; i++) {
+		if (new->list[i].err > 0) {
+			/* Subvol might be marked for decommission
+			   with EINVAL, or some other serious error
+			   marked with positive errno.
+			*/
+			continue;
+                }
+
+                max_overlap = 0;
+                max_overlap_idx = i;
+                for (j = (i + 1); j < new->cnt; ++j) {
+                        if (new->list[j].err > 0) {
+			        /* Subvol might be marked for decommission
+			        with EINVAL, or some other serious error
+			        marked with positive errno.
+			        */
+			        continue;
+                        }
+                        /* Calculate the overlap now. */
+                        curr_overlap = OV_ENTRY(i,i) + OV_ENTRY(j,j);
+                        /* Calculate the overlap after the proposed swap. */
+                        overlap = OV_ENTRY(i,j) + OV_ENTRY(j,i);
+                        /* Are we better than status quo? */
+                        if (overlap > curr_overlap) {
+                                overlap -= curr_overlap;
+                                /* Are we better than the previous choice? */
+                                if (overlap > max_overlap) {
+                                        max_overlap = overlap;
+                                        max_overlap_idx = j;
+                                }
+                        }
+                }
+
+		if (max_overlap_idx != i) {
+ 			dht_layout_range_swap (new, i, max_overlap_idx);
+                        /* Need to swap the table values too. */
+                        for (j = 0; j < old->cnt; ++j) {
+                                overlap = OV_ENTRY(i,j);
+                                OV_ENTRY(i,j) = OV_ENTRY(max_overlap_idx,j);
+                                OV_ENTRY(max_overlap_idx,j) = overlap;
+                        }
+                }
+	}
+}
+
+
 dht_layout_t *
 dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
                              dht_layout_t *layout)
 {
-        uint32_t      chunk        = 0;
-        uint32_t      start        = 0;
-        uint32_t      stop         = 0;
-        uint32_t      overlap      = 0;
-        uint32_t      max_overlap  = 0;
-        uint32_t      chunk_begin  = 0;
-        int           count        = 0;
-        int           cnt          = 0;
         int           i            = 0;
-        int           j            = 0;
-        int           k            = 0;
-        int           loop_cnt     = 0;
-        int           start_subvol = 0;
-        int          *fix_array    = NULL;
         xlator_t     *this         = NULL;
         dht_layout_t *new_layout   = NULL;
         dht_conf_t   *priv         = NULL;
         dht_local_t  *local        = NULL;
+        uint32_t      subvol_down  = 0;
+        int           ret          = 0;
 
         this  = frame->this;
         priv  = this->private;
         local = frame->local;
 
-        count = cnt = dht_get_layout_count (this, layout, 0);
-
-        chunk = ((unsigned long) 0xffffffff) / ((cnt) ? cnt : 1);
-
-        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
-
-        fix_array = GF_CALLOC (sizeof (int), layout->cnt, gf_common_mt_char);
-        if (!fix_array) {
-                /* No fix, use the existing layout itself */
+        if (layout->type == DHT_HASH_TYPE_DM_USER) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "leaving %s alone",
+                        loc->path);
                 goto done;
         }
 
@@ -554,98 +751,33 @@ dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
         if (!new_layout)
                 goto done;
 
-        for (i = 0; i < new_layout->cnt; i++) {
-                /* TODO: fix this in layout_alloc() itself */
-                new_layout->list[i].err = -ENOENT;
-                if (i < layout->cnt)
-                        new_layout->list[i].xlator = layout->list[i].xlator;
-        }
+        /* If a subvolume is down, do not re-write the layout. */
+        ret = dht_layout_anomalies (this, loc, layout, NULL, NULL, NULL,
+                                    &subvol_down, NULL, NULL);
 
-        /* Check if there are any overlap in layout, and give the proper fix */
-        for (i = 0; i < layout->cnt; i++) {
-                /* No need to fix if 'err' is not '-1' */
-                if (layout->list[i].err != -1)
-                        continue;
-
-                /* If already existing layout is having no range, skip it */
-                start = layout->list[i].start;
-                stop  = layout->list[i].stop;
-                if ((stop - start) == 0)
-                        continue;
-
-                max_overlap = 0;
-
-                /* 'j' is used as starting point of each chunk */
-                for (j = 1; j <= count; j++) {
-                        /* if chunk is already used, don't use it again */
-                        for (k = 0; k < i; k++)
-                                if (j == fix_array[k])
-                                        break;
-                        if (k < i)
-                                continue;
-
-                        overlap = dht_find_overlap (i, (j-1), start, stop, chunk);
-                        if (max_overlap < overlap) {
-                                max_overlap = overlap;
-                                fix_array[i] = j;
-                        }
-                }
-
-                /* If we have any overlap, then use that itself as new
-                   layout for the subvolume */
-                if (fix_array[i]) {
-                        chunk_begin = chunk * (fix_array[i] - 1);
-                        new_layout->list[i].err = -1;
-                        DHT_SET_LAYOUT_RANGE (new_layout, i, chunk_begin,
-                                              chunk, cnt, loc->path);
-                        /* make sure to give (max - 1) as 'stop' range,
-                           if it is last chunk */
-                        if (fix_array[i] == count)
-                                new_layout->list[i].stop = 0xffffffff;
-                        if (--cnt == 0)
-                                goto done;
-
-                }
+        if (subvol_down || (ret == -1)) {
+                gf_log (this->name, GF_LOG_WARNING, "%u subvolume(s) are down"
+                        ". Skipping fix layout.", subvol_down);
+                GF_FREE (new_layout);
+                return NULL;
         }
 
-        /* Now, look for layouts which are not having any overlaps
-           and give it a fix */
-        for (loop_cnt = 0, i = start_subvol; loop_cnt < new_layout->cnt;
-             i++, loop_cnt++) {
-                if (i == new_layout->cnt)
-                        i = 0;
-
-                /* If 'fix_array[i]' is set, the layout is already fixed. */
-                if (fix_array[i])
-                        continue;
+        for (i = 0; i < new_layout->cnt; i++) {
+		if (layout->list[i].err != ENOSPC)
+			new_layout->list[i].err = layout->list[i].err;
+		else
+			new_layout->list[i].err = -1;
 
-                if (layout->list[i].err != -1) {
-                        new_layout->list[i].err = layout->list[i].err;
-                        continue;
-                }
+		new_layout->list[i].xlator = layout->list[i].xlator;
+        }
 
-                for (k = 1; k <= count; k++) {
-                        for (j = 0; j < new_layout->cnt; j++) {
-                                if (k == fix_array[j])
-                                        break;
-                        }
-                        /* Didn't find any of the list begining with 'k' */
-                        if (j == new_layout->cnt)
-                                break;
-                }
+	/* First give it a layout as though it is a new directory. This
+	   ensures rotation to kick in */
+        dht_layout_sort_volname (new_layout);
+	dht_selfheal_layout_new_directory (frame, loc, new_layout);
 
-                fix_array[i] = k;
-                chunk_begin = (k - 1) * chunk;
-                new_layout->list[i].err = -1;
-                DHT_SET_LAYOUT_RANGE (new_layout, i, chunk_begin, chunk, cnt,
-                                      loc->path);
-                /* make sure to give (max - 1) as 'stop' range,
-                   if it is last chunk */
-                if (k == count)
-                        new_layout->list[i].stop = 0xffffffff;
-                if (--cnt == 0)
-                        goto done;
-        }
+	/* Now selectively re-assign ranges only when it helps */
+	dht_selfheal_layout_maximize_overlap (frame, loc, new_layout, layout);
 
 done:
         if (new_layout) {
@@ -659,7 +791,7 @@ done:
                 local->layout = new_layout;
         }
 
-        return new_layout;
+        return local->layout;
 }
 
 
@@ -683,9 +815,11 @@ dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
 
         start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
 
+        /* clear out the range, as we are re-computing here */
+        DHT_RESET_LAYOUT_RANGE (layout);
         for (i = start_subvol; i < layout->cnt; i++) {
                 err = layout->list[i].err;
-                if (err == -1) {
+                if (err == -1 || err == ENOENT) {
                         DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
                                              cnt, loc->path);
                         if (--cnt == 0) {
@@ -698,7 +832,7 @@ dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
 
         for (i = 0; i < start_subvol; i++) {
                 err = layout->list[i].err;
-                if (err == -1) {
+                if (err == -1 || err == ENOENT) {
                         DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
                                              cnt, loc->path);
                         if (--cnt == 0) {
@@ -717,35 +851,17 @@ int
 dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
                           dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-        xlator_t    *this = NULL;
         dht_local_t *local = NULL;
-        int          missing = -1;
-        int          down = -1;
-        int          holes = -1;
+        uint32_t     holes = 0;
         int          ret = -1;
         int          i = -1;
-        int          overlaps = -1;
+        uint32_t     overlaps = 0;
 
-        this = frame->this;
-        conf = this->private;
         local = frame->local;
 
-        missing = local->selfheal.missing;
-        down = local->selfheal.down;
         holes = local->selfheal.hole_cnt;
         overlaps = local->selfheal.overlaps_cnt;
 
-        if ((missing + down) == conf->subvolume_cnt) {
-                dht_selfheal_layout_new_directory (frame, loc, layout);
-                ret = 0;
-        }
-
-        if (holes <= down) {
-                /* the down subvol might fill up the holes */
-                ret = 0;
-        }
-
         if (holes || overlaps) {
                 dht_selfheal_layout_new_directory (frame, loc, layout);
                 ret = 0;
@@ -797,6 +913,9 @@ dht_fix_directory_layout (call_frame_t *frame,
 
         /* No layout sorting required here */
         tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
+        if (!tmp_layout) {
+                return -1;
+        }
         dht_fix_dir_xattr (frame, &local->loc, tmp_layout);
 
         return 0;
@@ -819,9 +938,8 @@ dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
         dht_layout_anomalies (this, loc, layout,
                               &local->selfheal.hole_cnt,
                               &local->selfheal.overlaps_cnt,
-                              &local->selfheal.missing,
-                              &local->selfheal.down,
-                              &local->selfheal.misc);
+                              NULL, &local->selfheal.down,
+                              &local->selfheal.misc, NULL);
 
         down     = local->selfheal.down;
         misc     = local->selfheal.misc;
@@ -880,3 +998,51 @@ dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
 
         return ret;
 }
+
+int
+dht_dir_attr_heal (void *data)
+{
+        call_frame_t    *frame = NULL;
+        dht_local_t     *local = NULL;
+        xlator_t        *subvol = NULL;
+        xlator_t        *this  = NULL;
+        dht_conf_t      *conf  = NULL;
+        int              call_cnt = 0;
+        int              ret   = -1;
+        int              i     = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", data, out);
+
+        frame = data;
+        local = frame->local;
+        this = frame->this;
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO ("dht", conf, out);
+
+        call_cnt = conf->subvolume_cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = conf->subvolumes[i];
+                if (!subvol || (subvol == dht_first_up_subvol (this)))
+                        continue;
+                ret = syncop_setattr (subvol, &local->loc, &local->stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                      NULL, NULL);
+                if (ret) {
+                        gf_log ("dht", GF_LOG_ERROR, "Failed to set uid/gid on"
+                                " %s on %s subvol (%s)", local->loc.path,
+                                subvol->name, strerror (-ret));
+                }
+        }
+out:
+        return 0;
+}
+
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data)
+{
+        DHT_STACK_DESTROY (sync_frame);
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
new file mode 100644
index 000000000..36c073973
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -0,0 +1,781 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+/* TODO: add NS locking */
+
+#include "statedump.h"
+#include "dht-common.h"
+
+/* TODO:
+   - use volumename in xattr instead of "dht"
+   - use NS locks
+   - handle all cases in self heal layout reconstruction
+   - complete linkfile selfheal
+*/
+struct volume_options options[];
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix)
+{
+
+        char    key[GF_DUMP_MAX_BUF_LEN];
+        int     i = 0;
+
+        if (!layout)
+                goto out;
+        if (!prefix)
+                goto out;
+
+        gf_proc_dump_build_key(key, prefix, "cnt");
+        gf_proc_dump_write(key, "%d", layout->cnt);
+        gf_proc_dump_build_key(key, prefix, "preset");
+        gf_proc_dump_write(key, "%d", layout->preset);
+        gf_proc_dump_build_key(key, prefix, "gen");
+        gf_proc_dump_write(key, "%d", layout->gen);
+        if (layout->type != IA_INVAL) {
+                gf_proc_dump_build_key(key, prefix, "inode type");
+                gf_proc_dump_write(key, "%d", layout->type);
+        }
+
+        if  (!IA_ISDIR (layout->type))
+                goto out;
+
+        for (i = 0; i < layout->cnt; i++) {
+                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
+                gf_proc_dump_write(key, "%d", layout->list[i].err);
+                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].start);
+                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].stop);
+                if (layout->list[i].xlator) {
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.type", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->type);
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.name", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->name);
+                }
+        }
+
+out:
+        return;
+}
+
+
+int32_t
+dht_priv_dump (xlator_t *this)
+{
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        char            key[GF_DUMP_MAX_BUF_LEN];
+        int             i = 0;
+        dht_conf_t      *conf = NULL;
+        int             ret = -1;
+
+        if (!this)
+                goto out;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        ret = TRY_LOCK(&conf->subvolume_lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
+                               this->name);
+        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                snprintf (key, sizeof (key), "subvolumes[%d]", i);
+                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                                   conf->subvolumes[i]->name);
+                if (conf->file_layouts && conf->file_layouts[i]){
+                        snprintf (key, sizeof (key), "file_layouts[%d]", i);
+                        dht_layout_dump(conf->file_layouts[i], key);
+                }
+                if (conf->dir_layouts && conf->dir_layouts[i]) {
+                        snprintf (key, sizeof (key), "dir_layouts[%d]", i);
+                        dht_layout_dump(conf->dir_layouts[i], key);
+                }
+                if (conf->subvolume_status) {
+
+                        snprintf (key, sizeof (key), "subvolume_status[%d]", i);
+                        gf_proc_dump_write(key, "%d",
+                                           (int)conf->subvolume_status[i]);
+                }
+
+        }
+
+        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+        gf_proc_dump_write("gen", "%d", conf->gen);
+        gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+	gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+
+        if (conf->du_stats) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (!conf->subvolume_status[i])
+                                continue;
+
+                        snprintf (key, sizeof (key), "subvolumes[%d]", i);
+                        gf_proc_dump_write (key, "%s",
+                                            conf->subvolumes[i]->name);
+
+                        snprintf (key, sizeof (key),
+                                  "du_stats[%d].avail_percent", i);
+                        gf_proc_dump_write (key, "%lf",
+                                            conf->du_stats[i].avail_percent);
+
+                        snprintf (key, sizeof (key), "du_stats[%d].avail_space",
+                                  i);
+                        gf_proc_dump_write (key, "%lu",
+                                            conf->du_stats[i].avail_space);
+
+                        snprintf (key, sizeof (key),
+                                  "du_stats[%d].avail_inodes", i);
+                        gf_proc_dump_write (key, "%lf",
+                                            conf->du_stats[i].avail_inodes);
+
+                        snprintf (key, sizeof (key), "du_stats[%d].log", i);
+                        gf_proc_dump_write (key, "%lu",
+                                            conf->du_stats[i].log);
+                }
+        }
+
+        if (conf->last_stat_fetch.tv_sec)
+                gf_proc_dump_write("last_stat_fetch", "%s",
+                                    ctime(&conf->last_stat_fetch.tv_sec));
+
+        UNLOCK(&conf->subvolume_lock);
+
+out:
+        return ret;
+}
+
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode)
+{
+        int             ret = -1;
+        dht_layout_t    *layout = NULL;
+
+        if (!this)
+                goto out;
+        if (!inode)
+                goto out;
+
+        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+
+        if ((ret != 0) || !layout)
+                return ret;
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+        dht_layout_dump(layout, "layout");
+
+out:
+        return ret;
+}
+
+void
+dht_fini (xlator_t *this)
+{
+        int         i = 0;
+        dht_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        conf = this->private;
+        this->private = NULL;
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf);
+        }
+out:
+        return;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+int
+dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
+                                 const char *bricks)
+{
+        int         i  = 0;
+        int         ret  = -1;
+        char       *tmpstr = NULL;
+        char       *dup_brick = NULL;
+        char       *node = NULL;
+
+        if (!conf || !bricks)
+                goto out;
+
+        dup_brick = gf_strdup (bricks);
+        node = strtok_r (dup_brick, ",", &tmpstr);
+        while (node) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (!strcmp (conf->subvolumes[i]->name, node)) {
+                                conf->decommissioned_bricks[i] =
+                                        conf->subvolumes[i];
+                                        conf->decommission_subvols_cnt++;
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "decommissioning subvolume %s",
+                                        conf->subvolumes[i]->name);
+                                break;
+                        }
+                }
+                if (i == conf->subvolume_cnt) {
+                        /* Wrong node given. */
+                        goto out;
+                }
+                node = strtok_r (NULL, ",", &tmpstr);
+        }
+
+        ret = 0;
+        conf->decommission_in_progress = 1;
+out:
+        GF_FREE (dup_brick);
+
+        return ret;
+}
+
+
+int
+dht_decommissioned_remove (xlator_t *this, dht_conf_t *conf)
+{
+        int         i  = 0;
+        int         ret  = -1;
+
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->decommissioned_bricks[i]) {
+                        conf->decommissioned_bricks[i] = NULL;
+                        conf->decommission_subvols_cnt--;
+                }
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+void
+dht_init_regex (xlator_t *this, dict_t *odict, char *name,
+                regex_t *re, gf_boolean_t *re_valid)
+{
+        char    *temp_str;
+
+        if (dict_get_str (odict, name, &temp_str) != 0) {
+                if (strcmp(name,"rsync-hash-regex")) {
+                        return;
+                }
+                temp_str = "^\\.(.+)\\.[^.]+$";
+        }
+
+        if (*re_valid) {
+                regfree(re);
+                *re_valid = _gf_false;
+        }
+
+        if (!strcmp(temp_str,"none")) {
+                return;
+        }
+
+        if (regcomp(re,temp_str,REG_EXTENDED) == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "using regex %s = %s", name, temp_str);
+                *re_valid = _gf_true;
+        }
+        else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "compiling regex %s failed", temp_str);
+        }
+}
+
+int
+dht_reconfigure (xlator_t *this, dict_t *options)
+{
+        dht_conf_t      *conf = NULL;
+        char            *temp_str = NULL;
+        gf_boolean_t     search_unhashed;
+        int              ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", options, out);
+
+        conf = this->private;
+        if (!conf)
+                return 0;
+
+        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean*/
+                if (strcasecmp (temp_str, "auto")) {
+                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
+                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                                       " lookup-unhashed reconfigured (%s)",
+                                       temp_str);
+                                conf->search_unhashed = search_unhashed;
+                        } else {
+                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
+                                       " lookup-unhashed should be boolean,"
+                                       " not (%s), defaulting to (%d)",
+                                       temp_str, conf->search_unhashed);
+                                //return -1;
+                                ret = -1;
+                                goto out;
+                        }
+                } else {
+                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                               " lookup-unhashed reconfigured auto ");
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+                }
+        }
+
+	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
+                          percent_or_size, out);
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100.0)
+                conf->disk_unit = 'p';
+
+	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
+                          percent, out);
+
+        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
+                          options, uint32, out);
+
+        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
+                          bool, out);
+        if (conf->defrag) {
+                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
+                                  options, bool, out);
+        }
+
+        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto out;
+        } else {
+                ret = dht_decommissioned_remove (this, conf);
+                if (ret == -1)
+                        goto out;
+        }
+
+        dht_init_regex (this, options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+gf_defrag_pattern_list_fill (xlator_t *this, gf_defrag_info_t *defrag, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *num = NULL;
+        char                  *pattern_str = NULL;
+        char                  *pattern = NULL;
+        gf_defrag_pattern_list_t *temp_list = NULL;
+        gf_defrag_pattern_list_t *pattern_list = NULL;
+
+        if (!this || !defrag || !data)
+                goto out;
+
+        /* Get the pattern for pattern list. "pattern:<optional-size>"
+         * eg: *avi, *pdf:10MB, *:1TB
+         */
+        pattern_str = strtok_r (data, ",", &tmp_str);
+        while (pattern_str) {
+                dup_str = gf_strdup (pattern_str);
+                pattern_list = GF_CALLOC (1, sizeof (gf_defrag_pattern_list_t),
+                                        1);
+                if (!pattern_list) {
+                        goto out;
+                }
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!pattern)
+                        goto out;
+                if (!num) {
+                        if (gf_string2bytesize(pattern, &pattern_list->size)
+                             == 0) {
+                                pattern = "*";
+                        }
+                } else if (gf_string2bytesize (num, &pattern_list->size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+                memcpy (pattern_list->path_pattern, pattern, strlen (dup_str));
+
+                if (!defrag->defrag_pattern)
+                        temp_list = NULL;
+                else
+                        temp_list = defrag->defrag_pattern;
+
+                pattern_list->next = temp_list;
+
+                defrag->defrag_pattern = pattern_list;
+                pattern_list = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                pattern_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (pattern_list);
+        GF_FREE (dup_str);
+
+        return ret;
+}
+
+int
+dht_init (xlator_t *this)
+{
+        dht_conf_t                      *conf           = NULL;
+        char                            *temp_str       = NULL;
+        int                              ret            = -1;
+        int                              i              = 0;
+        gf_defrag_info_t                *defrag         = NULL;
+        int                              cmd            = 0;
+        char                            *node_uuid      = NULL;
+
+
+        GF_VALIDATE_OR_GOTO ("dht", this, err);
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Distribute needs more than one subvolume");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile");
+        }
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
+        if (!conf) {
+                goto err;
+        }
+
+        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
+
+        if (cmd) {
+                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
+                                    gf_defrag_info_mt);
+
+                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
+
+                LOCK_INIT (&defrag->lock);
+
+                defrag->is_exiting = 0;
+
+                conf->defrag = defrag;
+
+                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "node-uuid not "
+                                "specified");
+                        goto err;
+                }
+
+                if (uuid_parse (node_uuid, defrag->node_uuid)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot parse "
+                                "glusterd node uuid");
+                        goto err;
+                }
+
+                defrag->cmd = cmd;
+
+                defrag->stats = _gf_false;
+        }
+
+        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean */
+                if (strcasecmp (temp_str, "auto"))
+                        gf_string2boolean (temp_str, &conf->search_unhashed);
+                else
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+
+        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
+                        err);
+
+        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
+
+	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
+			err);
+
+        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
+                        err);
+
+        conf->dir_spread_cnt = conf->subvolume_cnt;
+        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
+                        uint32, err);
+
+        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
+                        bool, err);
+
+        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
+
+        if (defrag) {
+                GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
+                if (dict_get_str (this->options, "rebalance-filter", &temp_str)
+                    == 0) {
+                        if (gf_defrag_pattern_list_fill (this, defrag, temp_str)
+                            == -1) {
+                                gf_log (this->name, GF_LOG_ERROR, "Cannot parse"
+                                        " rebalance-filter (%s)", temp_str);
+                                goto err;
+                        }
+                }
+        }
+
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100)
+                conf->disk_unit = 'p';
+
+        ret = dht_init_subvolumes (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto err;
+        }
+
+        dht_init_regex (this, this->options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, this->options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = dht_layouts_init (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        LOCK_INIT (&conf->subvolume_lock);
+        LOCK_INIT (&conf->layout_lock);
+
+        conf->gen = 1;
+
+        this->local_pool = mem_pool_new (dht_local_t, 512);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
+
+        GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);
+        gf_asprintf (&conf->link_xattr_name, "%s."DHT_LINKFILE_STR,
+                     conf->xattr_name);
+        gf_asprintf (&conf->wild_xattr_name, "%s*", conf->xattr_name);
+        if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+                goto err;
+        }
+
+        this->private = conf;
+
+        return 0;
+
+err:
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf->du_stats);
+
+                GF_FREE (conf->defrag);
+
+                GF_FREE (conf->xattr_name);
+                GF_FREE (conf->link_xattr_name);
+                GF_FREE (conf->wild_xattr_name);
+
+                GF_FREE (conf);
+        }
+
+        return -1;
+}
+
+
+struct volume_options options[] = {
+        { .key  = {"lookup-unhashed"},
+          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
+                    "on", "off"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
+          .description = "This option if set to ON, does a lookup through "
+          "all the sub-volumes, in case a lookup didn't return any result "
+          "from the hash subvolume. If set to OFF, it does not do a lookup "
+          "on the remaining subvolumes."
+        },
+        { .key  = {"min-free-disk"},
+          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+          .default_value = "10%",
+          .description = "Percentage/Size of disk space, after which the "
+          "process starts balancing out the cluster, and logs will appear "
+          "in log files",
+        },
+	{ .key  = {"min-free-inodes"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .default_value = "5%",
+          .description = "after system has only N% of inodes, warnings "
+          "starts to appear in log files",
+        },
+        { .key = {"unhashed-sticky-bit"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+        { .key = {"use-readdirp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option if set to ON, forces the use of "
+          "readdirp, and hence also displays the stats of the files."
+        },
+        { .key = {"assert-no-child-down"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON, in the event of "
+          "CHILD_DOWN, will call exit."
+        },
+        { .key  = {"directory-layout-spread"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "Specifies the directory layout spread. Takes number "
+                         "of subvolumes as default value."
+        },
+        { .key  = {"decommissioned-bricks"},
+          .type = GF_OPTION_TYPE_ANY,
+          .description = "This option if set to ON, decommissions "
+          "the brick, so that no new data is allowed to be created "
+          "on that brick."
+        },
+        { .key  = {"rebalance-cmd"},
+          .type = GF_OPTION_TYPE_INT,
+        },
+        { .key = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+        { .key = {"rebalance-stats"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON displays and logs the "
+          " time taken for migration of each file, during the rebalance "
+          "process. If set to OFF, the rebalance logs will only display the "
+          "time spent in each directory."
+        },
+        { .key = {"readdir-optimize"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON enables the optimization "
+          "that allows DHT to requests non-first subvolumes to filter out "
+          "directory entries."
+        },
+        { .key = {"rsync-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by rsync, to prevent relocation when the "
+          "file is renamed."
+        },
+        { .key = {"extra-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by an application, to prevent relocation when "
+          "the file is renamed."
+        },
+        { .key = {"rebalance-filter"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key = {"xattr-name"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "trusted.glusterfs.dht",
+          .description = "Base for extended attributes used by this "
+          "translator instance, to avoid conflicts with others above or "
+          "below it."
+        },
+
+        /* NUFA option */
+        { .key  = {"local-volume-name"},
+          .type = GF_OPTION_TYPE_XLATOR
+        },
+
+        /* switch option */
+        { .key  = {"pattern.switch.case"},
+          .type = GF_OPTION_TYPE_ANY
+        },
+
+        { .key  = {NULL} },
+};
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 96382bf16..fc0ca2f77 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -23,486 +14,15 @@
 #include "config.h"
 #endif
 
-/* TODO: add NS locking */
-
 #include "statedump.h"
 #include "dht-common.h"
 
-/* TODO:
-   - use volumename in xattr instead of "dht"
-   - use NS locks
-   - handle all cases in self heal layout reconstruction
-   - complete linkfile selfheal
-*/
-struct volume_options options[];
-
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix)
-{
-
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", layout, out);
-        GF_VALIDATE_OR_GOTO ("dht", prefix, out);
-
-        gf_proc_dump_build_key(key, prefix, "cnt");
-        gf_proc_dump_write(key, "%d", layout->cnt);
-        gf_proc_dump_build_key(key, prefix, "preset");
-        gf_proc_dump_write(key, "%d", layout->preset);
-        gf_proc_dump_build_key(key, prefix, "gen");
-        gf_proc_dump_write(key, "%d", layout->gen);
-        gf_proc_dump_build_key(key, prefix, "type");
-        gf_proc_dump_write(key, "%d", layout->type);
-
-        for (i = 0; i < layout->cnt; i++) {
-                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
-                gf_proc_dump_write(key, "%d", layout->list[i].err);
-                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].start);
-                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].stop);
-                if (layout->list[i].xlator) {
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.type", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->type);
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.name", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->name);
-                }
-        }
-
-out:
-        return;
-}
-
-
-int32_t
-dht_priv_dump (xlator_t *this)
-{
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        int             i = 0;
-        dht_conf_t      *conf = NULL;
-        int             ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-
-        if (!conf)
-                return -1;
-
-        ret = TRY_LOCK(&conf->subvolume_lock);
-
-        if (ret != 0) {
-                gf_log("", GF_LOG_WARNING, "Unable to lock dht subvolume %s",
-                       this->name);
-                return ret;
-        }
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
-                               this->name);
-        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                sprintf (key, "subvolumes[%d]", i);
-                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
-                                   conf->subvolumes[i]->name);
-                if (conf->file_layouts && conf->file_layouts[i]){
-                        sprintf (key, "file_layouts[%d]", i);
-                        dht_layout_dump(conf->file_layouts[i], key);
-                }
-                if (conf->dir_layouts && conf->dir_layouts[i]) {
-                        sprintf (key, "dir_layouts[%d]", i);
-                        dht_layout_dump(conf->dir_layouts[i], key);
-                }
-                if (conf->subvolume_status) {
-
-                        sprintf (key, "subvolume_status[%d]", i);
-                        gf_proc_dump_write(key, "%d",
-                                           (int)conf->subvolume_status[i]);
-                }
-
-        }
-
-        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
-        gf_proc_dump_write("gen", "%d", conf->gen);
-        gf_proc_dump_write("min_free_disk", "%lu", conf->min_free_disk);
-	gf_proc_dump_write("min_free_inodes", "%lu", conf->min_free_inodes);
-        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
-        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
-        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
-        if (conf ->du_stats) {
-                gf_proc_dump_write("du_stats.avail_percent", "%lf",
-                                   conf->du_stats->avail_percent);
-                gf_proc_dump_write("du_stats.avail_space", "%lu",
-                                   conf->du_stats->avail_space);
-		gf_proc_dump_write("du_stats.avail_inodes", "%lf",
-                                   conf->du_stats->avail_inodes);
-                gf_proc_dump_write("du_stats.log", "%lu", conf->du_stats->log);
-        }
-        gf_proc_dump_write("last_stat_fetch", "%s", ctime(&conf->last_stat_fetch.tv_sec));
-
-        UNLOCK(&conf->subvolume_lock);
-
-out:
-        return ret;
-}
-
-int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode)
-{
-        int             ret = -1;
-        dht_layout_t    *layout = NULL;
-        uint64_t        tmp_layout = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", inode, out);
-
-        ret = inode_ctx_get (inode, this, &tmp_layout);
-
-        if (ret != 0)
-                return ret;
-
-        layout = (dht_layout_t *)(long)tmp_layout;
-
-        if (!layout)
-                return -1;
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
-        dht_layout_dump(layout, "layout");
-
-out:
-        return ret;
-}
-
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int              ret = -1;
-        va_list          ap;
-        dict_t          *output = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-
-        if (!data)
-                goto out;
-
-        va_start (ap, data);
-        output = va_arg (ap, dict_t*);
-
-        ret = dht_notify (this, event, data, output);
-
-out:
-        return ret;
-}
-
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-        this->private = NULL;
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
-        }
-out:
-        return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-int
-dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
-                                 const char *bricks)
-{
-        int         i  = 0;
-        int         ret  = -1;
-        char       *tmpstr = NULL;
-        char       *dup_brick = NULL;
-        char       *node = NULL;
-
-        if (!conf || !bricks)
-                goto out;
-
-        dup_brick = gf_strdup (bricks);
-        node = strtok_r (dup_brick, ",", &tmpstr);
-        while (node) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!strcmp (conf->subvolumes[i]->name, node)) {
-                                conf->decommissioned_bricks[i] =
-                                        conf->subvolumes[i];
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "decommissioning subvolume %s",
-                                        conf->subvolumes[i]->name);
-                                break;
-                        }
-                }
-                if (i == conf->subvolume_cnt) {
-                        /* Wrong node given. */
-                        goto out;
-                }
-                node = strtok_r (NULL, ",", &tmpstr);
-        }
-
-        ret = 0;
-        conf->decommission_in_progress = 1;
-out:
-        if (dup_brick)
-                GF_FREE (dup_brick);
-
-        return ret;
-}
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        dht_conf_t      *conf = NULL;
-        char            *temp_str = NULL;
-        gf_boolean_t     search_unhashed;
-        int              ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", options, out);
-
-        conf = this->private;
-        if (!conf)
-                return 0;
-
-        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean*/
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-                                       " lookup-unhashed reconfigured (%s)",
-                                       temp_str);
-                                conf->search_unhashed = search_unhashed;
-                        } else {
-                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
-                                       " lookup-unhashed should be boolean,"
-                                       " not (%s), defaulting to (%d)",
-                                       temp_str, conf->search_unhashed);
-                                //return -1;
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-                               " lookup-unhashed reconfigured auto ");
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-                }
-        }
-
-	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
-                          percent_or_size, out);
-	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
-                          percent, out);
-        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
-                          options, uint32, out);
-
-        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-int
-init (xlator_t *this)
-{
-        dht_conf_t                      *conf           = NULL;
-        char                            *temp_str       = NULL;
-        int                              ret            = -1;
-        int                              i              = 0;
-        gf_defrag_info_t                *defrag         = NULL;
-        int                              cmd            = 0;
-        char                            *node_uuid      = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Distribute needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
-
-        if (cmd) {
-                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
-                                    gf_defrag_info_mt);
-
-                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
-
-                LOCK_INIT (&defrag->lock);
-
-                defrag->is_exiting = 0;
-
-                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "node-uuid not "
-                                "specified");
-                        goto err;
-                }
-
-                if (uuid_parse (node_uuid, defrag->node_uuid)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot parse "
-                                "glusterd node uuid");
-                        goto err;
-                }
-
-                defrag->cmd = cmd;
-
-                conf->defrag = defrag;
-        }
-
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
-
-        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
-                        err);
-
-        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
-
-	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
-			err);
-
-        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
-                        err);
-
-        conf->dir_spread_cnt = conf->subvolume_cnt;
-        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
-                        uint32, err);
-
-        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
-                        bool, err);
-
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
-
-        conf->gen = 1;
-
-        /* Create 'syncop' environment */
-	conf->env = syncenv_new (0);
-        if (!conf->env) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create sync environment %s",
-                        strerror (errno));
-                goto err;
-        }
-
-        this->local_pool = mem_pool_new (dht_local_t, 512);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
-        }
-
-        this->private = conf;
-
-        return 0;
-
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
-        return -1;
-}
-
+class_methods_t class_methods = {
+        .init           = dht_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 struct xlator_fops fops = {
         .lookup      = dht_lookup,
@@ -550,6 +70,9 @@ struct xlator_fops fops = {
         .fxattrop    = dht_fxattrop,
         .setattr     = dht_setattr,
         .fsetattr    = dht_fsetattr,
+	.fallocate   = dht_fallocate,
+	.discard     = dht_discard,
+        .zerofill    = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
@@ -563,51 +86,4 @@ struct xlator_cbks cbks = {
 //      .releasedir = dht_releasedir,
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "on",
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-          .default_value = "10%",
-          .description = "Percentage/Size of disk space that must be "
-                         "kept free."
-        },
-	{ .key  = {"min-free-inodes"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "5%",
-          .description = "Percentage inodes that must be "
-                         "kept free."
-        },
-        { .key = {"unhashed-sticky-bit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key = {"assert-no-child-down"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key  = {"directory-layout-spread"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key  = {"decommissioned-bricks"},
-          .type = GF_OPTION_TYPE_ANY,
-        },
-        { .key  = {"rebalance-cmd"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-
-        { .key  = {NULL} },
-};
+;
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index 951fe4e04..e934acdf0 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -27,6 +18,8 @@
 
 /* TODO: all 'TODO's in dht.c holds good */
 
+extern struct volume_options options[];
+
 int
 nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
@@ -44,7 +37,6 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           call_cnt    = 0;
         int           ret         = 0;
 
-
         conf  = this->private;
 
         prev  = cookie;
@@ -62,7 +54,8 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
@@ -141,7 +134,7 @@ out:
 
 err:
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, NULL);
+                          inode, stbuf, xattr, postparent);
         return 0;
 }
 
@@ -211,7 +204,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                  *       revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -232,7 +225,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
         } else {
         do_fresh_lookup:
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -241,7 +234,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                 }
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht.linkto", 256);
+                                       conf->link_xattr_name, 256);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -260,7 +253,8 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
@@ -269,7 +263,7 @@ nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int op_ret, int op_errno,
                                  inode_t *inode, struct iatt *stbuf,
                                  struct iatt *preparent,
-                                 struct iatt *postparent)
+                                 struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
@@ -280,21 +274,21 @@ nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
 
         STACK_WIND (frame, dht_create_cbk,
                     local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->fd,
-                    local->params);
+                    &local->loc, local->flags, local->mode, local->umask,
+                    local->fd, local->params);
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
 int
 nufa_create (call_frame_t *frame, xlator_t *this,
              loc_t *loc, int32_t flags, mode_t mode,
-             fd_t *fd, dict_t *params)
+             mode_t umask, fd_t *fd, dict_t *params)
 {
         dht_local_t *local = NULL;
         dht_conf_t  *conf  = NULL;
@@ -329,7 +323,8 @@ nufa_create (call_frame_t *frame, xlator_t *this,
         if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
                 avail_subvol =
                         dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
+                                                        (xlator_t *)conf->private,
+                                                        local);
         }
 
         if (subvol != avail_subvol) {
@@ -337,11 +332,10 @@ nufa_create (call_frame_t *frame, xlator_t *this,
                 local->params = dict_ref (params);
                 local->mode = mode;
                 local->flags = flags;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     nufa_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
@@ -350,14 +344,14 @@ nufa_create (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -366,34 +360,39 @@ int
 nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, inode_t *inode,
                          struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
         if (op_ret >= 0) {
-                STACK_WIND (frame, dht_newfile_cbk,
-                            local->cached_subvol,
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
                             local->cached_subvol->fops->mknod,
                             &local->loc, local->mode, local->rdev,
-                            local->params);
+                            local->umask, local->params);
 
                 return 0;
         }
-
+err:
         WIPE (postparent);
         WIPE (preparent);
 
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent);
+                          inode, stbuf, preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 nufa_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
         dht_local_t *local = NULL;
         dht_conf_t  *conf  = NULL;
@@ -429,7 +428,8 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
         if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
                 avail_subvol =
                         dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
+                                                        (xlator_t *)conf->private,
+                                                        local);
         }
 
         if (avail_subvol != subvol) {
@@ -437,10 +437,11 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
 
                 local->params = dict_ref (params);
                 local->mode = mode;
+                local->umask = umask;
                 local->rdev = rdev;
                 local->cached_subvol = avail_subvol;
 
-                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk,
+                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk, this,
                                      avail_subvol, subvol, loc);
                 return 0;
         }
@@ -448,218 +449,185 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->mknod,
-                    loc, mode, rdev, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
 
 
-int
-notify (xlator_t *this, int event, void *data, ...)
+gf_boolean_t
+same_first_part (char *str1, char term1, char *str2, char term2)
 {
-        int ret = -1;
-
-        ret = dht_notify (this, event, data);
-
-        return ret;
+        gf_boolean_t    ended1;
+        gf_boolean_t    ended2;
+
+        for (;;) {
+                ended1 = ((*str1 == '\0') || (*str1 == term1));
+                ended2 = ((*str2 == '\0') || (*str2 == term2));
+                if (ended1 && ended2) {
+                        return _gf_true;
+                }
+                if (ended1 || ended2 || (*str1 != *str2)) {
+                        return _gf_false;
+                }
+                ++str1;
+                ++str2;
+        }
 }
 
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
+typedef struct nufa_args {
+        xlator_t    *this;
+        char        *volname;
+        gf_boolean_t addr_match;
+} nufa_args_t;
 
-        conf = this->private;
+static void
+nufa_find_local_brick (xlator_t *xl, void *data)
+{
+        nufa_args_t     *args = data;
+        xlator_t        *this = args->this;
+        char            *local_volname = args->volname;
+        gf_boolean_t    addr_match = args->addr_match;
+        char            *brick_host = NULL;
+        dht_conf_t      *conf = this->private;
+        int             ret = -1;
+
+        /*This means a local subvol was already found. We pick the first brick
+         * that is local*/
+        if (conf->private)
+                return;
+
+        if (strcmp (xl->name, local_volname) == 0) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using specified subvol %s",
+                        local_volname);
+                return;
+        }
 
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+        if (!addr_match)
+                return;
 
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
+        ret = dict_get_str (xl->options, "remote-host", &brick_host);
+        if ((ret == 0) &&
+            (gf_is_same_address (local_volname, brick_host) ||
+             gf_is_local_addr (brick_host))) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using the first local "
+                        "subvol %s", xl->name);
+                return;
+        }
 
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
+}
 
-                GF_FREE (conf);
-        }
+static void
+nufa_to_dht (xlator_t *this)
+{
+        GF_ASSERT (this);
+        GF_ASSERT (this->fops);
 
-        return;
+        this->fops->lookup = dht_lookup;
+        this->fops->create = dht_create;
+        this->fops->mknod  = dht_mknod;
 }
 
 int
-init (xlator_t *this)
+nufa_find_local_subvol (xlator_t *this,
+                        void (*fn) (xlator_t *each, void* data), void *data)
 {
-        dht_conf_t    *conf = NULL;
-        xlator_list_t *trav = NULL;
-        data_t        *data = NULL;
-        char          *local_volname = NULL;
-        char          *temp_str = NULL;
-        int            ret = -1;
-        int            i = 0;
-        char           my_hostname[256];
-        uint32_t       temp_free_disk = 0;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "NUFA needs more than one subvolume");
+        int             ret = -1;
+        dht_conf_t      *conf = this->private;
+        xlator_list_t   *trav = NULL;
+        xlator_t        *parent = NULL;
+        xlator_t        *candidate = NULL;
+
+        xlator_foreach_depth_first (this, fn, data);
+        if (!conf->private) {
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't find a local "
+                        "brick");
                 return -1;
         }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf),
-                          gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
+        candidate = conf->private;
+        trav = candidate->parents;
+        while (trav) {
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+                parent = trav->xlator;
+                if (strcmp (parent->type, "cluster/nufa") == 0) {
+                        gf_log (this->name, GF_LOG_INFO, "Found local subvol, "
+                                "%s", candidate->name);
+                        ret = 0;
+                        conf->private = candidate;
+                        break;
+                }
 
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
+                candidate = parent;
+                trav = parent->parents;
         }
 
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
+        return ret;
+}
 
-        conf->gen = 1;
+int
+nufa_init (xlator_t *this)
+{
+        data_t        *data = NULL;
+        char          *local_volname = NULL;
+        int            ret = -1;
+        char           my_hostname[256];
+        gf_boolean_t   addr_match = _gf_false;
+        nufa_args_t    args = {0, };
 
-        local_volname = "localhost";
-        ret = gethostname (my_hostname, 256);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not find hostname (%s)",
-                        strerror (errno));
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
 
-        if (ret == 0)
-                local_volname = my_hostname;
-
-        data = dict_get (this->options, "local-volume-name");
-        if (data) {
+        if ((data = dict_get (this->options, "local-volume-name"))) {
                 local_volname = data->data;
-        }
 
-        trav = this->children;
-        while (trav) {
-                if (strcmp (trav->xlator->name, local_volname) == 0)
-                        break;
-                trav = trav->next;
-        }
-
-        if (!trav) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not find subvolume named '%s'. "
-                        "Please define volume with the name as the hostname "
-                        "or override it with 'option local-volume-name'",
-                        local_volname);
-                goto err;
-        }
-        /* The volume specified exists */
-        conf->private = trav->xlator;
-
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                            &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
-        }
+        } else {
+                addr_match = _gf_true;
+                local_volname = "localhost";
+                ret = gethostname (my_hostname, 256);
+                if (ret == 0)
+                        local_volname = my_hostname;
 
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                goto err;
-        }
+                else
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "could not find hostname (%s)",
+                                strerror (errno));
 
-        /* Create 'syncop' environment */
-	conf->env = syncenv_new (0);
-        if (!conf->env) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create sync environment %s",
-                        strerror (errno));
-                goto err;
         }
 
-        this->local_pool = mem_pool_new (dht_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
+        args.this = this;
+        args.volname = local_volname;
+        args.addr_match = addr_match;
+        ret = nufa_find_local_subvol (this, nufa_find_local_brick, &args);
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Unable to find local subvolume, switching "
+                        "to dht mode");
+                nufa_to_dht (this);
         }
-
-        this->private = conf;
-
         return 0;
+}
 
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
 
-        return -1;
-}
+class_methods_t class_methods = {
+        .init           = nufa_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 
 struct xlator_fops fops = {
@@ -706,19 +674,3 @@ struct xlator_fops fops = {
 struct xlator_cbks cbks = {
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {"local-volume-name"},
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index 1451e011e..2717ce975 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -31,6 +22,8 @@
 #include <fnmatch.h>
 #include <string.h>
 
+extern struct volume_options options[];
+
 struct switch_sched_array {
         xlator_t *xl;
         int32_t   eligible;
@@ -76,29 +69,37 @@ get_switch_matching_subvol (const char *path, dht_conf_t *conf,
         struct switch_struct *cond      = NULL;
         struct switch_struct *trav      = NULL;
         char                 *pathname  = NULL;
-        int                   idx     = 0;
+        int                   idx       = 0;
+        xlator_t             *subvol    = NULL;
 
         cond = conf->private;
+        subvol = hashed_subvol;
         if (!cond)
-                return hashed_subvol;
+                goto out;
 
-        trav = cond;
         pathname = gf_strdup (path);
+        if (!pathname)
+                goto out;
+
+        trav = cond;
         while (trav) {
                 if (fnmatch (trav->path_pattern,
                              pathname, FNM_NOESCAPE) == 0) {
                         for (idx = 0; idx < trav->num_child; idx++) {
                                 if (trav->array[idx].xl == hashed_subvol)
-                                        return hashed_subvol;
+                                        goto out;
                         }
                         idx = trav->node_index++;
                         trav->node_index %= trav->num_child;
-                        return trav->array[idx].xl;
+                        subvol = trav->array[idx].xl;
+                        goto out;
                 }
                 trav = trav->next;
         }
+out:
         GF_FREE (pathname);
-        return hashed_subvol;
+
+        return subvol;
 }
 
 
@@ -136,7 +137,8 @@ switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
@@ -290,11 +292,11 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
                  * attribute, revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
 
                 for (i = 0; i < layout->cnt; i++) {
                         subvol = layout->list[i].xlator;
@@ -309,18 +311,18 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
         } else {
         do_fresh_lookup:
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht.linkto", 256);
+                                       conf->link_xattr_name, 256);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht.linkto");
+                                "failed to set dict value for %s",
+                                conf->link_xattr_name);
 
                 if (!hashed_subvol) {
                         gf_log (this->name, GF_LOG_DEBUG,
@@ -366,7 +368,8 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -375,7 +378,7 @@ switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                    xlator_t *this, int op_ret, int op_errno,
                                    inode_t *inode, struct iatt *stbuf,
                                    struct iatt *preparent,
-                                   struct iatt *postparent)
+                                   struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
@@ -386,21 +389,21 @@ switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
 
         STACK_WIND (frame, dht_create_cbk,
                     local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->fd,
-                    local->params);
+                    &local->loc, local->flags, local->mode, local->umask,
+                    local->fd, local->params);
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
 int
 switch_create (call_frame_t *frame, xlator_t *this,
                loc_t *loc, int32_t flags, mode_t mode,
-               fd_t *fd, dict_t *params)
+               mode_t umask, fd_t *fd, dict_t *params)
 {
         dht_local_t *local = NULL;
         dht_conf_t  *conf  = NULL;
@@ -434,18 +437,18 @@ switch_create (call_frame_t *frame, xlator_t *this,
         avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
         if (dht_is_subvol_filled (this, avail_subvol)) {
                 avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
         }
 
         if (subvol != avail_subvol) {
                 /* create a link file instead of actual file */
                 local->mode = mode;
                 local->flags = flags;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     switch_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, switch_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
@@ -454,14 +457,14 @@ switch_create (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -470,31 +473,36 @@ int
 switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            int op_ret, int op_errno, inode_t *inode,
                            struct iatt *stbuf, struct iatt *preparent,
-                           struct iatt *postparent)
+                           struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
         if (op_ret >= 0) {
-                STACK_WIND (frame, dht_newfile_cbk,
-                            local->cached_subvol,
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
                             local->cached_subvol->fops->mknod,
                             &local->loc, local->mode, local->rdev,
-                            local->params);
+                            local->umask, local->params);
 
                 return 0;
         }
-
+err:
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent);
+                          inode, stbuf, preparent, postparent, xdata);
         return 0;
 }
 
 
 int
-switch_mknod (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *params)
 {
         dht_local_t *local = NULL;
         dht_conf_t  *conf  = NULL;
@@ -529,7 +537,8 @@ switch_mknod (call_frame_t *frame, xlator_t *this,
         avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
         if (dht_is_subvol_filled (this, avail_subvol)) {
                 avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
         }
 
         if (avail_subvol != subvol) {
@@ -537,46 +546,36 @@ switch_mknod (call_frame_t *frame, xlator_t *this,
 
                 local->params = dict_ref (params);
                 local->mode = mode;
+                local->umask = umask;
                 local->rdev = rdev;
                 local->cached_subvol = avail_subvol;
 
                 dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
-                                     avail_subvol, subvol, loc);
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->mknod,
-                    loc, mode, rdev, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
 
         return 0;
 
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
         DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+                          NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
 
 
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int ret = -1;
-
-        ret = dht_notify (this, event, data);
-
-        return ret;
-}
-
 void
-fini (xlator_t *this)
+switch_fini (xlator_t *this)
 {
-        int                   i = 0;
         dht_conf_t           *conf = NULL;
         struct switch_struct *trav = NULL;
         struct switch_struct *prev = NULL;
@@ -587,30 +586,14 @@ fini (xlator_t *this)
                 trav = (struct switch_struct *)conf->private;
                 conf->private = NULL;
                 while (trav) {
-                        if (trav->array)
-                                GF_FREE (trav->array);
+                        GF_FREE (trav->array);
                         prev = trav;
                         trav = trav->next;
                         GF_FREE (prev);
                 }
-
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
         }
 
-        return;
+        dht_fini(this);
 }
 
 int
@@ -670,8 +653,10 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                 dup_str = gf_strdup (switch_str);
                 switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
                                         gf_switch_mt_switch_struct);
-                if (!switch_opt)
+                if (!switch_opt) {
+                        GF_FREE (dup_str);
                         goto err;
+                }
 
                 pattern = strtok_r (dup_str, ":", &tmp_str1);
                 childs = strtok_r (NULL, ":", &tmp_str1);
@@ -681,9 +666,11 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                                 "for all the unconfigured child nodes,"
                                 " hence neglecting current option");
                         switch_str = strtok_r (NULL, ";", &tmp_str);
+                        GF_FREE (switch_opt);
                         GF_FREE (dup_str);
                         continue;
                 }
+                GF_FREE (dup_str);
                 memcpy (switch_opt->path_pattern, pattern, strlen (pattern));
                 if (childs) {
                         dup_childs = gf_strdup (childs);
@@ -740,7 +727,6 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                                 "option in unify volume. Exiting");
                         goto err;
                 }
-                GF_FREE (dup_str);
 
                 /* Link it to the main structure */
                 if (switch_buf) {
@@ -753,6 +739,7 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                         /* First entry */
                         switch_buf = switch_opt;
                 }
+                switch_opt = NULL;
                 switch_str = strtok_r (NULL, ";", &tmp_str);
         }
 
@@ -809,19 +796,20 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                         /* First entry */
                         switch_buf = switch_opt;
                 }
+                switch_opt = NULL;
         }
         /* */
         conf->private = switch_buf;
 
         return 0;
 err:
+        GF_FREE (switch_buf_array);
+        GF_FREE (switch_opt);
+
         if (switch_buf) {
-                if (switch_buf_array)
-                        GF_FREE (switch_buf_array);
                 trav = switch_buf;
                 while (trav) {
-                        if (trav->array)
-                                GF_FREE (trav->array);
+                        GF_FREE (trav->array);
                         switch_opt = trav;
                         trav = trav->next;
                         GF_FREE (switch_opt);
@@ -831,68 +819,18 @@ err:
 }
 
 
-int
-init (xlator_t *this)
+int32_t
+switch_init (xlator_t *this)
 {
         dht_conf_t            *conf = NULL;
         data_t                *data = NULL;
-        char                  *temp_str = NULL;
         int                    ret = -1;
-        int                    i = 0;
-        uint32_t               temp_free_disk = 0;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "SWITCH needs more than one subvolume");
-                return -1;
-        }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_switch_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
-
-        conf->unhashed_sticky_bit = 0;
-        if (dict_get_str (this->options, "unhashed-sticky-bit",
-                          &temp_str) == 0) {
-                gf_string2boolean (temp_str, &conf->unhashed_sticky_bit);
-        }
-
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                            &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
+        conf = this->private;
 
         data = dict_get (this->options, "pattern.switch.case");
         if (data) {
@@ -903,72 +841,23 @@ init (xlator_t *this)
                 }
         }
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
-
-        conf->gen = 1;
-
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_switch_mt_dht_du_t);
-        if (!conf->du_stats) {
-                goto err;
-        }
-
-        /* Create 'syncop' environment */
-	conf->env = syncenv_new (0);
-        if (!conf->env) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create sync environment %s",
-                        strerror (errno));
-                goto err;
-        }
-
-        this->local_pool = mem_pool_new (dht_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
-        }
-
         this->private = conf;
-
         return 0;
 
 err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
+        dht_fini(this);
         return -1;
 }
 
 
+class_methods_t class_methods = {
+        .init           = switch_init,
+        .fini           = switch_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
+
+
 struct xlator_fops fops = {
         .lookup      = switch_lookup,
         .create      = switch_create,
@@ -1013,19 +902,3 @@ struct xlator_fops fops = {
 struct xlator_cbks cbks = {
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {"pattern.switch.case"},
-          .type = GF_OPTION_TYPE_ANY
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_mock.c b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
new file mode 100644
index 000000000..aa19ddc57
--- /dev/null
+++ b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
@@ -0,0 +1,63 @@
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "dht-common.h"
+#include "byte-order.h"
+
+int
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+{
+    return 0;
+}
+
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+    return 0;
+}
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+    return 0;
+}
+
+int
+dict_get_ptr (dict_t *this, char *key, void **ptr)
+{
+    return 0;
+}
+
+int
+dict_get_ptr_and_len (dict_t *this, char *key, void **ptr, int *len)
+{
+    return 0;
+}
+
+int _gf_log (const char *domain, const char *file,
+             const char *function, int32_t line, gf_loglevel_t level,
+             const char *fmt, ...)
+{
+    return 0;
+}
+
+int _gf_log_callingfn (const char *domain, const char *file,
+                       const char *function, int32_t line, gf_loglevel_t level,
+                       const char *fmt, ...)
+{
+    return 0;
+}
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
new file mode 100644
index 000000000..b5233d235
--- /dev/null
+++ b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
@@ -0,0 +1,124 @@
+/*
+  Copyright (c) 2008-2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "dht-common.h"
+#include "logging.h"
+#include "xlator.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <inttypes.h>
+#include <cmockery/pbc.h>
+#include <cmockery/cmockery.h>
+
+/*
+ * Helper functions
+ */
+
+static xlator_t *
+helper_xlator_init(uint32_t num_types)
+{
+    xlator_t *xl;
+    int i, ret;
+
+    REQUIRE(num_types > 0);
+
+    xl = test_calloc(1, sizeof(xlator_t));
+    assert_non_null(xl);
+    xl->mem_acct.num_types = num_types;
+    xl->mem_acct.rec = test_calloc(num_types, sizeof(struct mem_acct_rec));
+    assert_non_null(xl->mem_acct.rec);
+
+    xl->ctx = test_calloc(1, sizeof(glusterfs_ctx_t));
+    assert_non_null(xl->ctx);
+
+    for (i = 0; i < num_types; i++) {
+            ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
+            assert_false(ret);
+    }
+
+    ENSURE(num_types == xl->mem_acct.num_types);
+    ENSURE(NULL != xl);
+
+    return xl;
+}
+
+static int
+helper_xlator_destroy(xlator_t *xl)
+{
+    int i, ret;
+
+    for (i = 0; i < xl->mem_acct.num_types; i++) {
+            ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
+            assert_int_equal(ret, 0);
+    }
+
+    free(xl->mem_acct.rec);
+    free(xl->ctx);
+    free(xl);
+    return 0;
+}
+
+/*
+ * Unit tests
+ */
+static void
+test_dht_layout_new(void **state)
+{
+    xlator_t *xl;
+    dht_layout_t *layout;
+    dht_conf_t   *conf;
+    int cnt;
+
+    expect_assert_failure(dht_layout_new(NULL, 0));
+    expect_assert_failure(dht_layout_new((xlator_t *)0x12345, -1));
+    xl = helper_xlator_init(10);
+
+    // xl->private is NULL
+    assert_null(xl->private);
+    cnt = 100;
+    layout = dht_layout_new(xl, cnt);
+    assert_non_null(layout);
+    assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
+    assert_int_equal(layout->cnt, cnt);
+    assert_int_equal(layout->ref, 1);
+    assert_int_equal(layout->gen, 0);
+    assert_int_equal(layout->spread_cnt, 0);
+    free(layout);
+
+    // xl->private is not NULL
+    cnt = 110;
+    conf = (dht_conf_t *)test_calloc(1, sizeof(dht_conf_t));
+    assert_non_null(conf);
+    conf->dir_spread_cnt = 12345;
+    conf->gen = -123;
+    xl->private = conf;
+
+    layout = dht_layout_new(xl, cnt);
+    assert_non_null(layout);
+    assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
+    assert_int_equal(layout->cnt, cnt);
+    assert_int_equal(layout->ref, 1);
+    assert_int_equal(layout->gen, conf->gen);
+    assert_int_equal(layout->spread_cnt, conf->dir_spread_cnt);
+    free(layout);
+
+    free(conf);
+    helper_xlator_destroy(xl);
+}
+
+int main(void) {
+    const UnitTest tests[] = {
+        unit_test(test_dht_layout_new),
+    };
+
+    return run_tests(tests, "xlator_dht_layout");
+}
diff --git a/xlators/cluster/ha/src/Makefile.am b/xlators/cluster/ha/src/Makefile.am
index 5f78a2965..5c1364b7f 100644
--- a/xlators/cluster/ha/src/Makefile.am
+++ b/xlators/cluster/ha/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = ha.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-ha_la_LDFLAGS = -module -avoidversion 
+ha_la_LDFLAGS = -module -avoid-version 
 
 ha_la_SOURCES = ha-helpers.c ha.c
 ha_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = ha.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/ha/src/ha-helpers.c b/xlators/cluster/ha/src/ha-helpers.c
index 1e4af1b62..19be1ed27 100644
--- a/xlators/cluster/ha/src/ha-helpers.c
+++ b/xlators/cluster/ha/src/ha-helpers.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include "xlator.h"
 #include "call-stub.h"
 #include "defaults.h"
diff --git a/xlators/cluster/ha/src/ha-mem-types.h b/xlators/cluster/ha/src/ha-mem-types.h
index 9bfb3972b..e5e97d237 100644
--- a/xlators/cluster/ha/src/ha-mem-types.h
+++ b/xlators/cluster/ha/src/ha-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __HA_MEM_TYPES_H__
 #define __HA_MEM_TYPES_H__
 
diff --git a/xlators/cluster/ha/src/ha.c b/xlators/cluster/ha/src/ha.c
index 38d4229d3..3eccb516b 100644
--- a/xlators/cluster/ha/src/ha.c
+++ b/xlators/cluster/ha/src/ha.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 /* generate errors randomly, code is simple now, better alogorithm
  * can be written to decide what error to be returned and when
  */
@@ -1876,13 +1866,9 @@ err:
         }
 
         if (hafdp) {
-                if (hafdp->fdstate) {
-                        GF_FREE (hafdp->fdstate);
-                }
+                GF_FREE (hafdp->fdstate);
 
-                if (hafdp->path) {
-                        GF_FREE (hafdp->path);
-                }
+                GF_FREE (hafdp->path);
 
                 GF_FREE (hafdp);
         }
diff --git a/xlators/cluster/ha/src/ha.h b/xlators/cluster/ha/src/ha.h
index 39b6851e7..e2ed7eaa6 100644
--- a/xlators/cluster/ha/src/ha.h
+++ b/xlators/cluster/ha/src/ha.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __HA_H_
 #define __HA_H_
 
diff --git a/xlators/cluster/map/src/Makefile.am b/xlators/cluster/map/src/Makefile.am
index 26e19137a..a278b05e2 100644
--- a/xlators/cluster/map/src/Makefile.am
+++ b/xlators/cluster/map/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = map.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-map_la_LDFLAGS = -module -avoidversion 
+map_la_LDFLAGS = -module -avoid-version 
 
 map_la_SOURCES = map.c map-helper.c
 map_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = map.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/map/src/map-helper.c b/xlators/cluster/map/src/map-helper.c
index 81212fcfd..851397b68 100644
--- a/xlators/cluster/map/src/map-helper.c
+++ b/xlators/cluster/map/src/map-helper.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2009-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/cluster/map/src/map-mem-types.h b/xlators/cluster/map/src/map-mem-types.h
index 669b93dc2..3e89f4736 100644
--- a/xlators/cluster/map/src/map-mem-types.h
+++ b/xlators/cluster/map/src/map-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __MAP_MEM_TYPES_H__
 #define __MAP_MEM_TYPES_H__
 
diff --git a/xlators/cluster/map/src/map.c b/xlators/cluster/map/src/map.c
index ead9da0b9..6150a33ce 100644
--- a/xlators/cluster/map/src/map.c
+++ b/xlators/cluster/map/src/map.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -2375,8 +2365,7 @@ fini (xlator_t *this)
 	priv = this->private;
 
 	if (priv) {
-		if (priv->xlarray)
-			GF_FREE (priv->xlarray);
+		GF_FREE (priv->xlarray);
 
 		trav_map = priv->map;
 		while (trav_map) {
diff --git a/xlators/cluster/map/src/map.h b/xlators/cluster/map/src/map.h
index bccac437c..7703a543e 100644
--- a/xlators/cluster/map/src/map.h
+++ b/xlators/cluster/map/src/map.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MAP_H__
 #define __MAP_H__
 
diff --git a/xlators/cluster/stripe/src/Makefile.am b/xlators/cluster/stripe/src/Makefile.am
index 0db3c9eeb..2d151422a 100644
--- a/xlators/cluster/stripe/src/Makefile.am
+++ b/xlators/cluster/stripe/src/Makefile.am
@@ -2,16 +2,19 @@
 xlator_LTLIBRARIES = stripe.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-stripe_la_LDFLAGS = -module -avoidversion
+stripe_la_LDFLAGS = -module -avoid-version
+
+stripe_la_SOURCES = stripe.c stripe-helpers.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-stripe_la_SOURCES = stripe.c $(top_builddir)/xlators/lib/src/libxlator.c
 stripe_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = stripe.h stripe-mem-types.h $(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/lib/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 CLEANFILES = 
 
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
new file mode 100644
index 000000000..a047d4a2e
--- /dev/null
+++ b/xlators/cluster/stripe/src/stripe-helpers.c
@@ -0,0 +1,677 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <fnmatch.h>
+
+#include "stripe.h"
+#include "byte-order.h"
+#include "mem-types.h"
+
+void
+stripe_local_wipe (stripe_local_t *local)
+{
+        if (!local)
+                goto out;
+
+        loc_wipe (&local->loc);
+        loc_wipe (&local->loc2);
+
+        if (local->fd)
+                fd_unref (local->fd);
+
+        if (local->inode)
+                inode_unref (local->inode);
+
+        if (local->xattr)
+                dict_unref (local->xattr);
+
+        if (local->xdata)
+                dict_unref (local->xdata);
+
+out:
+        return;
+}
+
+
+
+int
+stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
+{
+        dict_t  *dst  = NULL;
+        int64_t *ptr  = 0, *size = NULL;
+        int32_t  ret  = -1;
+
+        dst = data;
+
+        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
+                ret = dict_get_bin (dst, key, (void **)&size);
+                if (ret < 0) {
+                        size = GF_CALLOC (1, sizeof (int64_t),
+                                          gf_common_mt_char);
+                        if (size == NULL) {
+                                gf_log ("stripe", GF_LOG_WARNING,
+                                        "memory allocation failed");
+                                goto out;
+                        }
+                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
+                        if (ret < 0) {
+                                gf_log ("stripe", GF_LOG_WARNING,
+                                        "stripe aggregate dict set failed");
+                                GF_FREE (size);
+                                goto out;
+                        }
+                }
+
+                ptr = data_to_bin (value);
+                if (ptr == NULL) {
+                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
+                        goto out;
+                }
+
+                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+        } else if (strcmp (key, GF_CONTENT_KEY)) {
+                /* No need to aggregate 'CONTENT' data */
+                ret = dict_set (dst, key, value);
+                if (ret)
+                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
+        }
+
+out:
+        return 0;
+}
+
+
+void
+stripe_aggregate_xattr (dict_t *dst, dict_t *src)
+{
+        if ((dst == NULL) || (src == NULL)) {
+                goto out;
+        }
+
+        dict_foreach (src, stripe_aggregate, dst);
+out:
+        return;
+}
+
+
+int32_t
+stripe_xattr_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
+{
+        int32_t              i     = 0;
+        int32_t              ret   = -1;
+        int32_t              len   = 0;
+        char                *sbuf  = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (!buffer || !local || !local->xattr_list)
+                goto out;
+
+        sbuf = buffer;
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        memcpy (buffer, xattr->xattr_value, len);
+                        buffer += len;
+                        *buffer++ = ' ';
+                }
+        }
+
+        *--buffer = '\0';
+        if (total)
+                *total = buffer - sbuf;
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int32_t
+stripe_free_xattr_str (stripe_local_t *local)
+{
+        int32_t              i     = 0;
+        int32_t              ret   = -1;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (!local || !local->xattr_list)
+                goto out;
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+
+                if (xattr && xattr->xattr_value)
+                        GF_FREE (xattr->xattr_value);
+        }
+
+        ret = 0;
+ out:
+        return ret;
+}
+
+
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz)
+{
+        int32_t              ret   = -1, i = 0, len = 0;
+        dict_t              *tmp1  = NULL, *tmp2 = NULL;
+        char                *buf   = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (xattr_serz == NULL) {
+                goto out;
+        }
+
+        tmp2 = dict_new ();
+
+        if (tmp2 == NULL) {
+                goto out;
+        }
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        ret = dict_reset (tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "dict_reset failed (%s)",
+                                        strerror (-ret));
+                        }
+
+                        ret = dict_unserialize (xattr->xattr_value,
+                                                xattr->xattr_len,
+                                                &tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_unserialize failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+
+                        tmp1 = dict_copy (tmp2, tmp1);
+                        if (tmp1 == NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_copy failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        len = dict_serialized_length (tmp1);
+        if (len > 0) {
+                buf = GF_CALLOC (1, len, gf_common_mt_dict_t);
+                if (buf == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_serialize (tmp1, buf);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "dict_serialize failed (%s)", strerror (-ret));
+                        GF_FREE(buf);
+                        ret = -1;
+                        goto out;
+                }
+
+                *xattr_serz = buf;
+        }
+
+        ret = 0;
+out:
+        if (tmp1 != NULL) {
+                dict_unref (tmp1);
+        }
+
+        if (tmp2 != NULL) {
+                dict_unref (tmp2);
+        }
+
+        return ret;
+}
+
+
+int32_t
+stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            char **xattr_serz)
+{
+        int      ret             = -1;
+        int32_t  padding         = 0;
+        int32_t  tlen            = 0;
+        char stripe_size_str[20] = {0,};
+        char    *pathinfo_serz   = NULL;
+
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
+                goto out;
+        }
+
+        (void) snprintf (stripe_size_str, 20, "%ld",
+                         (local->fctx) ? local->fctx->stripe_size : 0);
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
+                + strlen (stripe_size_str) + 7;
+        local->xattr_total_len += (padding + 2);
+
+        pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
+                                   gf_common_mt_char);
+        if (!pathinfo_serz)
+                goto out;
+
+        /* xlator info */
+        (void) sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ",
+                        this->name, stripe_size_str);
+
+        ret = stripe_xattr_aggregate (pathinfo_serz + padding, local, &tlen);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot aggregate pathinfo list");
+                GF_FREE(pathinfo_serz);
+                goto out;
+        }
+
+        *(pathinfo_serz + padding + tlen) = ')';
+        *(pathinfo_serz + padding + tlen + 1) = '\0';
+
+        *xattr_serz = pathinfo_serz;
+
+        ret = 0;
+ out:
+        return ret;
+}
+
+/**
+ * stripe_get_matching_bs - Get the matching block size for the given path.
+ */
+int32_t
+stripe_get_matching_bs (const char *path, stripe_private_t *priv)
+{
+        struct stripe_options *trav       = NULL;
+        uint64_t               block_size = 0;
+
+        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
+        GF_VALIDATE_OR_GOTO ("stripe", path, out);
+
+        LOCK (&priv->lock);
+        {
+                block_size = priv->block_size;
+                trav = priv->pattern;
+                while (trav) {
+                        if (!fnmatch (trav->path_pattern, path, FNM_NOESCAPE)) {
+                                block_size = trav->block_size;
+                                break;
+                        }
+                        trav = trav->next;
+                }
+        }
+        UNLOCK (&priv->lock);
+
+out:
+        return block_size;
+}
+
+int32_t
+stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
+                   dict_t *dict)
+{
+        char            key[256]       = {0,};
+        data_t         *data            = NULL;
+        int32_t         index           = 0;
+        stripe_private_t *priv          = NULL;
+
+        priv = this->private;
+
+
+        if (!local->fctx) {
+                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
+                                         gf_stripe_mt_stripe_fd_ctx_t);
+                if (!local->fctx) {
+                        local->op_errno = ENOMEM;
+                        local->op_ret = -1;
+                        goto out;
+                }
+
+                local->fctx->static_array = 0;
+        }
+        /* Stripe block size */
+        sprintf (key, "trusted.%s.stripe-size", this->name);
+        data = dict_get (dict, key);
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-size");
+                goto out;
+        } else {
+                if (!local->fctx->stripe_size) {
+                        local->fctx->stripe_size =
+                                     data_to_int64 (data);
+                }
+
+                if (local->fctx->stripe_size != data_to_int64 (data)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "stripe-size mismatch in blocks");
+                        local->xattr_self_heal_needed = 1;
+                }
+        }
+
+        /* Stripe count */
+        sprintf (key, "trusted.%s.stripe-count", this->name);
+        data = dict_get (dict, key);
+
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-count");
+                goto out;
+        }
+        if (!local->fctx->xl_array) {
+                local->fctx->stripe_count = data_to_int32 (data);
+                if (!local->fctx->stripe_count) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error with stripe-count xattr");
+                        local->op_ret   = -1;
+                        local->op_errno = EIO;
+                        goto out;
+                }
+
+                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
+                                                   sizeof (xlator_t *),
+                                                   gf_stripe_mt_xlator_t);
+
+                if (!local->fctx->xl_array) {
+                        local->op_errno = ENOMEM;
+                        local->op_ret   = -1;
+                        goto out;
+                }
+        }
+        if (local->fctx->stripe_count != data_to_int32 (data)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error with stripe-count xattr (%d != %d)",
+                        local->fctx->stripe_count, data_to_int32 (data));
+                local->op_ret   = -1;
+                local->op_errno = EIO;
+                goto out;
+        }
+
+        /* index */
+        sprintf (key, "trusted.%s.stripe-index", this->name);
+        data = dict_get (dict, key);
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-index");
+                goto out;
+        }
+        index = data_to_int32 (data);
+        if (index > priv->child_count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error with stripe-index xattr (%d)", index);
+                local->op_ret   = -1;
+                local->op_errno = EIO;
+                goto out;
+        }
+        if (local->fctx->xl_array) {
+                if (!local->fctx->xl_array[index])
+                        local->fctx->xl_array[index] = prev->this;
+        }
+
+	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
+	data = dict_get(dict, key);
+	if (!data) {
+		/*
+		 * The file was probably created prior to coalesce support.
+		 * Assume non-coalesce mode for this file to maintain backwards
+		 * compatibility.
+		 */
+		gf_log(this->name, GF_LOG_DEBUG, "missing stripe-coalesce "
+			"attr, assume non-coalesce mode");
+		local->fctx->stripe_coalesce = 0;
+	} else {
+		local->fctx->stripe_coalesce = data_to_int32(data);
+	}
+
+
+out:
+        return 0;
+}
+
+int32_t
+stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
+                            uint32_t stripe_count, uint32_t stripe_index,
+			    uint32_t stripe_coalesce)
+{
+        char            key[256]       = {0,};
+        int32_t         ret             = -1;
+
+        sprintf (key, "trusted.%s.stripe-size", this->name);
+        ret = dict_set_int64 (dict, key, stripe_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+        sprintf (key, "trusted.%s.stripe-count", this->name);
+        ret = dict_set_int32 (dict, key, stripe_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+        sprintf (key, "trusted.%s.stripe-index", this->name);
+        ret = dict_set_int32 (dict, key, stripe_index);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
+	ret = dict_set_int32(dict, key, stripe_coalesce);
+	if (ret) {
+		gf_log(this->name, GF_LOG_WARNING,
+			"failed to set %s in xattr_req_dict", key);
+		goto out;
+	}
+out:
+        return ret;
+}
+
+
+static int
+set_default_block_size (stripe_private_t *priv, char *num)
+{
+
+        int             ret = -1;
+        GF_VALIDATE_OR_GOTO ("stripe", THIS, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, num, out);
+
+
+        if (gf_string2bytesize (num, &priv->block_size) != 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "invalid number format \"%s\"", num);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+
+}
+
+
+int
+set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *stripe_str = NULL;
+        char                  *pattern = NULL;
+        char                  *num = NULL;
+        struct stripe_options *temp_stripeopt = NULL;
+        struct stripe_options *stripe_opt = NULL;
+
+        if (!this || !priv || !data)
+                goto out;
+
+        /* Get the pattern for striping.
+           "option block-size *avi:10MB" etc */
+        stripe_str = strtok_r (data, ",", &tmp_str);
+        while (stripe_str) {
+                dup_str = gf_strdup (stripe_str);
+                stripe_opt = GF_CALLOC (1, sizeof (struct stripe_options),
+                                        gf_stripe_mt_stripe_options);
+                if (!stripe_opt) {
+                        goto out;
+                }
+
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!num) {
+                        num = pattern;
+                        pattern = "*";
+                        ret = set_default_block_size (priv, num);
+                        if (ret)
+                                goto out;
+                }
+                if (gf_string2bytesize (num, &stripe_opt->block_size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+
+                if (stripe_opt->block_size < STRIPE_MIN_BLOCK_SIZE) {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
+                                "%s. Should be atleast %llu bytes", num,
+                                STRIPE_MIN_BLOCK_SIZE);
+                        goto out;
+                }
+                if (stripe_opt->block_size % 512) {
+                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
+                                " be a multiple of 512 bytes", num);
+                        goto out;
+                }
+
+                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "block-size : pattern %s : size %"PRId64,
+                        stripe_opt->path_pattern, stripe_opt->block_size);
+
+                if (priv->pattern)
+                        temp_stripeopt = NULL;
+                else
+                        temp_stripeopt = priv->pattern;
+
+                stripe_opt->next = temp_stripeopt;
+
+                priv->pattern = stripe_opt;
+                stripe_opt = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                stripe_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+
+        GF_FREE (dup_str);
+
+        GF_FREE (stripe_opt);
+
+        return ret;
+}
+
+int32_t
+stripe_iatt_merge (struct iatt *from, struct iatt *to)
+{
+        if (to->ia_size < from->ia_size)
+                to->ia_size = from->ia_size;
+        if (to->ia_mtime < from->ia_mtime)
+                to->ia_mtime = from->ia_mtime;
+        if (to->ia_ctime < from->ia_ctime)
+                to->ia_ctime = from->ia_ctime;
+        if (to->ia_atime < from->ia_atime)
+                to->ia_atime = from->ia_atime;
+        return 0;
+}
+
+off_t
+coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count)
+{
+	size_t line_size = 0;
+	uint64_t stripe_num = 0;
+	off_t coalesced_offset = 0;
+
+	line_size = stripe_size * stripe_count;
+	stripe_num = offset / line_size;
+
+	coalesced_offset = (stripe_num * stripe_size) +
+		(offset % stripe_size);
+
+	return coalesced_offset;
+}
+
+off_t
+uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
+		int stripe_index)
+{
+        uint64_t nr_full_stripe_chunks = 0, mod = 0;
+
+        if (!size)
+                return size;
+
+	/*
+	 * Estimate the number of fully written stripes from the
+	 * local file size. Each stripe_size chunk corresponds to
+	 * a stripe.
+	 */
+        nr_full_stripe_chunks = (size / stripe_size) * stripe_count;
+        mod = size % stripe_size;
+
+        if (!mod) {
+                /*
+		 * There is no remainder, thus we could have overestimated
+		 * the size of the file in terms of chunks. Trim the number
+		 * of chunks by the following stripe members and leave it
+		 * up to those nodes to respond with a larger size (if
+		 * necessary).
+		 */
+                nr_full_stripe_chunks -= stripe_count -
+                        (stripe_index + 1);
+                size = nr_full_stripe_chunks * stripe_size;
+        } else {
+		/*
+		 * There is a remainder and thus we own the last chunk of the
+		 * file. Add the preceding stripe members of the final stripe
+		 * along with the remainder to calculate the exact size.
+		 */
+                nr_full_stripe_chunks += stripe_index;
+                size = nr_full_stripe_chunks * stripe_size + mod;
+        }
+
+        return size;
+}
+
diff --git a/xlators/cluster/stripe/src/stripe-mem-types.h b/xlators/cluster/stripe/src/stripe-mem-types.h
index d3b3db752..e9ac9cf46 100644
--- a/xlators/cluster/stripe/src/stripe-mem-types.h
+++ b/xlators/cluster/stripe/src/stripe-mem-types.h
@@ -1,21 +1,11 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -26,10 +16,11 @@
 
 enum gf_stripe_mem_types_ {
         gf_stripe_mt_iovec = gf_common_mt_end + 1,
-        gf_stripe_mt_readv_replies,
+        gf_stripe_mt_stripe_replies,
         gf_stripe_mt_stripe_fd_ctx_t,
         gf_stripe_mt_char,
         gf_stripe_mt_int8_t,
+        gf_stripe_mt_int32_t,
         gf_stripe_mt_xlator_t,
         gf_stripe_mt_stripe_private_t,
         gf_stripe_mt_stripe_options,
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
index 5b1610724..79e80b513 100644
--- a/xlators/cluster/stripe/src/stripe.c
+++ b/xlators/cluster/stripe/src/stripe.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /**
@@ -41,217 +32,10 @@
 
 struct volume_options options[];
 
-void
-stripe_local_wipe (stripe_local_t *local)
-{
-        if (!local)
-                goto out;
-
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
-
-        if (local->fd)
-                fd_unref (local->fd);
-
-        if (local->inode)
-                inode_unref (local->inode);
-
-        if (local->xattr)
-                dict_unref (local->xattr);
-
-        if (local->dict)
-                dict_unref (local->dict);
-
-out:
-        return;
-}
-
-/**
- * stripe_get_matching_bs - Get the matching block size for the given path.
- */
-int32_t
-stripe_get_matching_bs (const char *path, struct stripe_options *opts,
-                        uint64_t default_bs)
-{
-        struct stripe_options *trav       = NULL;
-        char                  *pathname   = NULL;
-        uint64_t               block_size = 0;
-
-        block_size = default_bs;
-
-        if (!path || !opts)
-                goto out;
-
-        /* FIXME: is a strdup really necessary? */
-        pathname = gf_strdup (path);
-        if (!pathname)
-                goto out;
-
-        trav = opts;
-        while (trav) {
-                if (!fnmatch (trav->path_pattern, pathname, FNM_NOESCAPE)) {
-                        block_size = trav->block_size;
-                        break;
-                }
-                trav = trav->next;
-        }
-
-        GF_FREE (pathname);
-
-out:
-        return block_size;
-}
-
-
-
-int32_t
-stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
-                   dict_t *dict)
-{
-        char            key[256]       = {0,};
-        data_t         *data            = NULL;
-        int32_t         index           = 0;
-        stripe_private_t *priv          = NULL;
-        int32_t         ret             = -1;
-
-        priv = this->private;
-
-
-        if (!local->fctx) {
-                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                         gf_stripe_mt_stripe_fd_ctx_t);
-                if (!local->fctx) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret = -1;
-                        goto out;
-                }
-
-                local->fctx->static_array = 0;
-        }
-        /* Stripe block size */
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-size");
-                goto out;
-        } else {
-                if (!local->fctx->stripe_size) {
-                        local->fctx->stripe_size =
-                                     data_to_int64 (data);
-                }
-
-                if (local->fctx->stripe_size != data_to_int64 (data)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "stripe-size mismatch in blocks");
-                        local->xattr_self_heal_needed = 1;
-                }
-        }
-
-        /* Stripe count */
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        data = dict_get (dict, key);
-
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-count");
-                goto out;
-        }
-        if (!local->fctx->xl_array) {
-                local->fctx->stripe_count = data_to_int32 (data);
-                if (!local->fctx->stripe_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error with stripe-count xattr");
-                        local->op_ret   = -1;
-                        local->op_errno = EIO;
-                        goto out;
-                }
-
-                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
-                                                   sizeof (xlator_t *),
-                                                   gf_stripe_mt_xlator_t);
-
-                if (!local->fctx->xl_array) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret   = -1;
-                        goto out;
-                }
-        }
-        if (local->fctx->stripe_count != data_to_int32 (data)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-count xattr (%d != %d)",
-                        local->fctx->stripe_count, data_to_int32 (data));
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-
-        /* index */
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-index");
-                goto out;
-        }
-        index = data_to_int32 (data);
-        if (index > priv->child_count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-index xattr (%d)", index);
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-        if (local->fctx->xl_array) {
-                if (!local->fctx->xl_array[index])
-                        local->fctx->xl_array[index] = prev->this;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
-                            uint32_t stripe_count, uint32_t stripe_index)
-{
-        char            key[256]       = {0,};
-        int32_t         ret             = -1;
-
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        ret = dict_set_int64 (dict, key, stripe_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        ret = dict_set_int32 (dict, key, stripe_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        ret = dict_set_int32 (dict, key, stripe_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-out:
-        return ret;
-}
-
 int32_t
 stripe_sh_chown_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *preop, struct iatt *postop)
+                     struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         int             callcnt = -1;
         stripe_local_t *local   = NULL;
@@ -280,7 +64,7 @@ int32_t
 stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, inode_t *inode,
                           struct iatt *buf, struct iatt *preparent,
-                          struct iatt *postparent)
+                          struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         call_frame_t    *prev = NULL;
@@ -295,7 +79,7 @@ stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         STACK_WIND (frame, stripe_sh_chown_cbk, prev->this,
                     prev->this->fops->setattr, &local->loc,
-                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID));
+                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
 
 out:
         return 0;
@@ -309,7 +93,7 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
         call_frame_t     *rframe = NULL;
         stripe_local_t   *rlocal = NULL;
         stripe_private_t *priv   = NULL;
-        dict_t           *dict   = NULL;
+        dict_t           *xdata  = NULL;
         int               ret    = 0;
 
         if (!local || !this || !frame) {
@@ -336,11 +120,11 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
         loc_copy (&rlocal->loc, &local->loc);
         memcpy (&rlocal->stbuf, &local->stbuf, sizeof (struct iatt));
 
-        dict = dict_new ();
-        if (!dict)
+        xdata = dict_new ();
+        if (!xdata)
                 goto out;
 
-        ret = dict_set_static_bin (dict, "gfid-req", local->stbuf.ia_gfid, 16);
+        ret = dict_set_static_bin (xdata, "gfid-req", local->stbuf.ia_gfid, 16);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to set gfid-req", local->loc.path);
@@ -351,97 +135,38 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
                                     trav->xlator, trav->xlator->fops->mknod,
                                     &local->loc,
                                     st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type), 0,
-                                    dict);
+                                                     local->stbuf.ia_type),
+                                    0, 0, xdata);
                 }
                 if (IA_ISDIR (local->stbuf.ia_type)) {
                         STACK_WIND (rframe, stripe_sh_make_entry_cbk,
                                     trav->xlator, trav->xlator->fops->mkdir,
-                                    &local->loc, st_mode_from_ia (local->stbuf.ia_prot,
-                                                                  local->stbuf.ia_type),
-                                    dict);
+                                    &local->loc,
+                                    st_mode_from_ia (local->stbuf.ia_prot,
+                                                     local->stbuf.ia_type),
+                                    0, xdata);
                 }
                 trav = trav->next;
         }
 
-        if (dict)
-                dict_unref (dict);
+        if (xdata)
+                dict_unref (xdata);
         return 0;
 
 out:
         if (rframe)
                 STRIPE_STACK_DESTROY (rframe);
-        if (dict)
-                dict_unref (dict);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
-void
-stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
-{
-        dict_t  *dst  = NULL;
-        int64_t *ptr  = 0, *size = NULL;
-        int32_t  ret  = -1;
-
-        dst = data;
-
-        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
-                ret = dict_get_bin (dst, key, (void **)&size);
-                if (ret < 0) {
-                        size = GF_CALLOC (1, sizeof (int64_t),
-                                          gf_common_mt_char);
-                        if (size == NULL) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "memory allocation failed");
-                                goto out;
-                        }
-                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
-                        if (ret < 0) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "stripe aggregate dict set failed");
-                                GF_FREE (size);
-                                goto out;
-                        }
-                }
-
-                ptr = data_to_bin (value);
-                if (ptr == NULL) {
-                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
-                        goto out;
-                }
-
-                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
-        } else if (strcmp (key, GF_CONTENT_KEY)) {
-                /* No need to aggregate 'CONTENT' data */
-                ret = dict_set (dst, key, value);
-                if (ret)
-                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
-        }
-
-out:
-        return;
-}
-
-
-void
-stripe_aggregate_xattr (dict_t *dst, dict_t *src)
-{
-        if ((dst == NULL) || (src == NULL)) {
-                goto out;
-        }
-
-        dict_foreach (src, stripe_aggregate, dst);
-out:
-        return;
-}
-
-
 int32_t
 stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
         int32_t         callcnt     = 0;
         stripe_local_t *local       = NULL;
@@ -479,7 +204,7 @@ stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = 0;
                         if (IA_ISREG (buf->ia_type)) {
                                 ret = stripe_ctx_handle (this, prev, local,
-                                                         dict);
+                                                         xdata);
                                 if (ret)
                                         gf_log (this->name, GF_LOG_ERROR,
                                                  "Error getting fctx info from"
@@ -490,26 +215,29 @@ stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->stbuf      = *buf;
                                 local->postparent = *postparent;
                                 local->inode = inode_ref (inode);
-                                local->dict = dict_ref (dict);
+                                if (xdata)
+                                        local->xdata = dict_ref (xdata);
                                 if (local->xattr) {
-                                        stripe_aggregate_xattr (local->dict,
+                                        stripe_aggregate_xattr (local->xdata,
                                                                 local->xattr);
                                         dict_unref (local->xattr);
                                         local->xattr = NULL;
                                 }
                         }
 
-                        if (!local->dict && !local->xattr) {
-                                local->xattr = dict_ref (dict);
-                        } else if (local->dict) {
-                                stripe_aggregate_xattr (local->dict, dict);
+                        if (!local->xdata && !local->xattr) {
+                                local->xattr = dict_ref (xdata);
+                        } else if (local->xdata) {
+                                stripe_aggregate_xattr (local->xdata, xdata);
                         } else if (local->xattr) {
-                                stripe_aggregate_xattr (local->xattr, dict);
+                                stripe_aggregate_xattr (local->xattr, xdata);
                         }
 
                         local->stbuf_blocks      += buf->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->postparent_size < postparent->ia_size)
@@ -547,7 +275,7 @@ stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STRIPE_STACK_UNWIND (lookup, frame, local->op_ret,
                                      local->op_errno, local->inode,
-                                     &local->stbuf, local->dict,
+                                     &local->stbuf, local->xdata,
                                      &local->postparent);
         }
 out:
@@ -556,7 +284,7 @@ out:
 
 int32_t
 stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xattr_req)
+               dict_t *xdata)
 {
         stripe_local_t   *local    = NULL;
         xlator_list_t    *trav     = NULL;
@@ -589,18 +317,29 @@ stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         if (tmpctx)
                 local->fctx = (stripe_fd_ctx_t*) (long)tmpctx;
 
-        if (xattr_req && dict_get (xattr_req, GF_CONTENT_KEY)) {
-                ret = dict_get_int64 (xattr_req, GF_CONTENT_KEY, &filesize);
+        /* quick-read friendly changes */
+        if (xdata && dict_get (xdata, GF_CONTENT_KEY)) {
+                ret = dict_get_int64 (xdata, GF_CONTENT_KEY, &filesize);
                 if (!ret && (filesize > priv->block_size))
-                        dict_del (xattr_req, GF_CONTENT_KEY);
+                        dict_del (xdata, GF_CONTENT_KEY);
         }
 
         /* get stripe-size xattr on lookup. This would be required for
          * open/read/write/pathinfo calls. Hence we send down the request
          * even when type == IA_INVAL */
-        if (xattr_req && (IA_ISREG (loc->inode->ia_type) ||
+
+	/*
+	 * We aren't guaranteed to have xdata here. We need the format info for
+	 * the file, so allocate xdata if necessary.
+	 */
+	if (!xdata)
+		xdata = dict_new();
+	else
+		xdata = dict_ref(xdata);
+
+        if (xdata && (IA_ISREG (loc->inode->ia_type) ||
             (loc->inode->ia_type == IA_INVAL))) {
-                ret = stripe_xattr_request_build (this, xattr_req, 8, 4, 4);
+                ret = stripe_xattr_request_build (this, xdata, 8, 4, 4, 0);
                 if (ret)
                         gf_log (this->name , GF_LOG_ERROR, "Failed to build"
                                 " xattr request for %s", loc->path);
@@ -612,11 +351,12 @@ stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_lookup_cbk, trav->xlator,
-                            trav->xlator->fops->lookup,
-                            loc, xattr_req);
+                            trav->xlator->fops->lookup, loc, xdata);
                 trav = trav->next;
         }
 
+	dict_unref(xdata);
+
         return 0;
 err:
         STRIPE_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
@@ -626,7 +366,7 @@ err:
 
 int32_t
 stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -661,6 +401,9 @@ stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         }
 
                         local->stbuf_blocks += buf->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                 }
@@ -677,18 +420,19 @@ stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 STRIPE_STACK_UNWIND (stat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf);
+                                     local->op_errno, &local->stbuf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -715,23 +459,30 @@ stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         frame->local = local;
         local->call_count = priv->child_count;
 
+	if (IA_ISREG(loc->inode->ia_type)) {
+		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         while (trav) {
                 STACK_WIND (frame, stripe_stat_cbk, trav->xlator,
-                            trav->xlator->fops->stat, loc);
+                            trav->xlator->fops->stat, loc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 
 err:
-        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf)
+                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         int32_t         callcnt = 0;
@@ -769,14 +520,14 @@ stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (!callcnt) {
                 STRIPE_STACK_UNWIND (statfs, frame, local->op_ret,
-                                     local->op_errno, &local->statvfs_buf);
+                                     local->op_errno, &local->statvfs_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
@@ -803,13 +554,13 @@ stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_statfs_cbk, trav->xlator,
-                            trav->xlator->fops->statfs, loc);
+                            trav->xlator->fops->statfs, loc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -818,7 +569,7 @@ err:
 int32_t
 stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -856,6 +607,9 @@ stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += prebuf->ia_blocks;
                         local->postbuf_blocks += postbuf->ia_blocks;
 
+			correct_file_size(prebuf, local->fctx, prev);
+			correct_file_size(postbuf, local->fctx, prev);
+
                         if (local->prebuf_size < prebuf->ia_size)
                                 local->prebuf_size = prebuf->ia_size;
 
@@ -878,19 +632,21 @@ stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STRIPE_STACK_UNWIND (truncate, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
 {
-        xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = EINVAL;
+	int		  i, eof_idx;
+	off_t		  dest_offset, tmp_offset;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -899,7 +655,6 @@ stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         VALIDATE_OR_GOTO (loc->inode, err);
 
         priv = this->private;
-        trav = this->children;
 
         if (priv->first_child_down) {
                 op_errno = ENOTCONN;
@@ -916,15 +671,55 @@ stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         frame->local = local;
         local->call_count = priv->child_count;
 
-        while (trav) {
-                STACK_WIND (frame, stripe_truncate_cbk, trav->xlator,
-                            trav->xlator->fops->truncate, loc, offset);
-                trav = trav->next;
-        }
+	inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	local->fctx = fctx;
+	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (!fctx->xl_array[i]) {
+			gf_log(this->name, GF_LOG_ERROR,
+				"no xlator at index %d", i);
+			op_errno = EINVAL;
+			goto err;
+		}
+
+		if (fctx->stripe_coalesce) {
+			/*
+	 		 * The node that owns EOF is truncated to the exact
+	 		 * coalesced offset. Nodes prior to this index should
+	 		 * be rounded up to the size of the complete stripe,
+	 		 * while nodes after this index should be rounded down
+			 * to the size of the previous stripe.
+			 */
+			if (i < eof_idx)
+				tmp_offset = roof(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else if (i > eof_idx)
+				tmp_offset = floor(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else
+				tmp_offset = offset;
+
+			dest_offset = coalesced_offset(tmp_offset,
+					fctx->stripe_size, fctx->stripe_count);
+		} else {
+			dest_offset = offset;
+		}
+
+		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
+			fctx->xl_array[i]->fops->truncate, loc, dest_offset,
+			NULL);
+	}
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -932,7 +727,7 @@ err:
 int32_t
 stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *preop, struct iatt *postop)
+                    struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -971,6 +766,9 @@ stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += preop->ia_blocks;
                         local->postbuf_blocks += postop->ia_blocks;
 
+			correct_file_size(preop, local->fctx, prev);
+			correct_file_size(postop, local->fctx, prev);
+
                         if (local->prebuf_size < preop->ia_size)
                                 local->prebuf_size = preop->ia_size;
                         if (local->postbuf_size < postop->ia_size)
@@ -992,7 +790,7 @@ stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STRIPE_STACK_UNWIND (setattr, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
@@ -1001,11 +799,12 @@ out:
 
 int32_t
 stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t	 *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -1035,28 +834,35 @@ stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 local->call_count = 1;
                 STACK_WIND (frame, stripe_setattr_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, NULL);
                 return 0;
         }
 
+	if (IA_ISREG(loc->inode->ia_type)) {
+		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_setattr_cbk,
                             trav->xlator, trav->xlator->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -1083,13 +889,13 @@ stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         while (trav) {
                 STACK_WIND (frame, stripe_setattr_cbk, trav->xlator,
-                            trav->xlator->fops->fsetattr, fd, stbuf, valid);
+                            trav->xlator->fops->fsetattr, fd, stbuf, valid, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1097,7 +903,8 @@ int32_t
 stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
                          struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent)
+                         struct iatt *prenewparent, struct iatt *postnewparent,
+                         dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -1134,6 +941,8 @@ stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->pre_buf.ia_blocks    += prenewparent->ia_blocks;
                         local->post_buf.ia_blocks   += postnewparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf.ia_size < buf->ia_size)
                                 local->stbuf.ia_size =  buf->ia_size;
 
@@ -1159,7 +968,7 @@ stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STRIPE_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                                      &local->stbuf, &local->preparent,
                                      &local->postparent,  &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
@@ -1169,7 +978,8 @@ int32_t
 stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
                          struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent)
+                         struct iatt *prenewparent, struct iatt *postnewparent,
+                         dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         xlator_list_t  *trav = NULL;
@@ -1200,24 +1010,25 @@ stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         while (trav) {
                 STACK_WIND (frame, stripe_stack_rename_cbk,
                             trav->xlator, trav->xlator->fops->rename,
-                            &local->loc, &local->loc2);
+                            &local->loc, &local->loc2, NULL);
                 trav = trav->next;
         }
         return 0;
 
 unwind:
         STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent);
+                             postoldparent, prenewparent, postnewparent, NULL);
         return 0;
 }
 
 int32_t
 stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
+               loc_t *newloc, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t	 *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -1242,27 +1053,35 @@ stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                 op_errno = ENOMEM;
                 goto err;
         }
+
+        frame->local = local;
+
         local->op_ret = -1;
         loc_copy (&local->loc, oldloc);
         loc_copy (&local->loc2, newloc);
 
         local->call_count = priv->child_count;
 
-        frame->local = local;
+	if (IA_ISREG(oldloc->inode->ia_type)) {
+		inode_ctx_get(oldloc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
 
         STACK_WIND (frame, stripe_first_rename_cbk, trav->xlator,
-                    trav->xlator->fops->rename, oldloc, newloc);
+                    trav->xlator->fops->rename, oldloc, newloc, NULL);
 
         return 0;
 err:
         STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+                             NULL, NULL, NULL);
         return 0;
 }
 int32_t
 stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                         int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t *local   = NULL;
         call_frame_t   *prev = NULL;
@@ -1287,10 +1106,10 @@ stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->postparent_blocks += postparent->ia_blocks;
 
         STRIPE_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
-                            &local->preparent, &local->postparent);
+                            &local->preparent, &local->postparent, xdata);
         return 0;
 out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1301,7 +1120,7 @@ out:
 int32_t
 stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -1337,17 +1156,19 @@ stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto out;
                 }
                 STACK_WIND(frame, stripe_first_unlink_cbk, FIRST_CHILD (this),
-                           FIRST_CHILD (this)->fops->unlink, &local->loc);
+                           FIRST_CHILD (this)->fops->unlink, &local->loc,
+                           local->xflag, local->xdata);
         }
         return 0;
 out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
 
 int32_t
-stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int xflag, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -1382,6 +1203,11 @@ stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
         local->op_ret = -1;
         loc_copy (&local->loc, loc);
+        local->xflag = xflag;
+
+        if (xdata)
+            local->xdata = dict_ref (xdata);
+
         frame->local = local;
         local->call_count = priv->child_count;
         trav = trav->next; /* Skip the first child */
@@ -1389,13 +1215,13 @@ stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         while (trav) {
                 STACK_WIND (frame, stripe_unlink_cbk,
                             trav->xlator, trav->xlator->fops->unlink,
-                            loc);
+                            loc, xflag, xdata);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1403,8 +1229,7 @@ err:
 int32_t
 stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent)
-
+                        struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
 
@@ -1431,10 +1256,10 @@ stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->postparent_blocks += postparent->ia_blocks;
 
         STRIPE_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                             &local->preparent, &local->postparent);
+                             &local->preparent, &local->postparent, xdata);
         return 0;
 err:
-        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
         return 0;
 
 }
@@ -1442,7 +1267,7 @@ err:
 int32_t
 stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -1474,16 +1299,16 @@ stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto out;
                 STACK_WIND (frame, stripe_first_rmdir_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->rmdir, &local->loc,
-                            local->flags);
+                            local->flags, NULL);
         }
         return 0;
 out:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 int32_t
-stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -1520,13 +1345,13 @@ stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 
         while (trav) {
                 STACK_WIND (frame, stripe_rmdir_cbk,  trav->xlator,
-                            trav->xlator->fops->rmdir, loc, flags);
+                            trav->xlator->fops->rmdir, loc, flags, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1535,7 +1360,7 @@ int32_t
 stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
                                     xlator_t *this, int32_t op_ret,
                                     int32_t op_errno, struct iatt *preparent,
-                                    struct iatt *postparent)
+                                    struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -1556,7 +1381,7 @@ stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
         if (!callcnt) {
                 STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1568,7 +1393,7 @@ out:
 int32_t
 stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
@@ -1607,7 +1432,7 @@ stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
                                             stripe_mknod_ifreg_fail_unlink_cbk,
                                             trav->xlator,
                                             trav->xlator->fops->unlink,
-                                            &local->loc);
+                                            &local->loc, 0, NULL);
                                 trav = trav->next;
                         }
                         return 0;
@@ -1615,7 +1440,7 @@ stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
 
                 STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1625,14 +1450,13 @@ int32_t
 stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, inode_t *inode,
                         struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent)
+                        struct iatt *postparent, dict_t *xdata)
 {
         int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         call_frame_t     *prev = NULL;
         xlator_list_t    *trav = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -1664,10 +1488,16 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         if (uuid_is_null (local->ia_gfid))
                                 uuid_copy (local->ia_gfid, buf->ia_gfid);
 
+			if (stripe_ctx_handle(this, prev, local, xdata))
+				gf_log(this->name, GF_LOG_ERROR,
+					"Error getting fctx info from dict");
+
                         local->stbuf_blocks += buf->ia_blocks;
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->preparent_size < preparent->ia_size)
@@ -1692,7 +1522,7 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                             stripe_mknod_ifreg_fail_unlink_cbk,
                                             trav->xlator,
                                             trav->xlator->fops->unlink,
-                                            &local->loc);
+                                            &local->loc, 0, NULL);
                                 trav = trav->next;
                         }
                         return 0;
@@ -1706,26 +1536,13 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->postparent.ia_size   = local->postparent_size;
                         local->stbuf.ia_size        = local->stbuf_size;
                         local->stbuf.ia_blocks      = local->stbuf_blocks;
-                        fctx = GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                          gf_stripe_mt_stripe_fd_ctx_t);
-                        if (!fctx) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unwind;
-                        }
-
-                        fctx->stripe_size  = local->stripe_size;
-                        fctx->stripe_count = priv->child_count;
-                        fctx->static_array = 1;
-                        fctx->xl_array = priv->xl_array;
                         inode_ctx_put (local->inode, this,
-                                       (uint64_t)(long)fctx);
+                                       (uint64_t)(long) local->fctx);
 
                 }
-unwind:
                 STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1736,7 +1553,7 @@ int32_t
 stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, inode_t *inode,
                         struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent)
+                        struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -1796,7 +1613,8 @@ stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                         ret = stripe_xattr_request_build (this, dict,
                                                           local->stripe_size,
-                                                          priv->child_count, i);
+                                                          priv->child_count, i,
+							  priv->coalesce);
                         if (ret)
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "Failed to build xattr request");
@@ -1807,7 +1625,7 @@ stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STACK_WIND (frame, stripe_mknod_ifreg_cbk,
                             trav->xlator, trav->xlator->fops->mknod,
-                            &local->loc, local->mode, local->rdev, dict);
+                            &local->loc, local->mode, local->rdev, 0, dict);
                 trav = trav->next;
                 i++;
 
@@ -1819,7 +1637,7 @@ stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
 
-       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL);
+       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL, NULL);
        return 0;
 }
 
@@ -1828,25 +1646,22 @@ int32_t
 stripe_single_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, inode_t *inode,
                          struct iatt *buf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
         STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, dict_t *params)
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
         stripe_private_t *priv           = NULL;
         stripe_local_t   *local          = NULL;
         int32_t           op_errno       = EINVAL;
         int32_t           i              = 0;
-        char              size_key[256]  = {0,};
-        char              index_key[256] = {0,};
-        char              count_key[256] = {0,};
         dict_t           *dict           = NULL;
         int               ret            = 0;
         int               need_unref     = 0;
@@ -1883,29 +1698,19 @@ stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 }
                 local->op_ret = -1;
                 local->op_errno = ENOTCONN;
-                local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                             priv->pattern,
-                                                             priv->block_size);
+                local->stripe_size = stripe_get_matching_bs (loc->path, priv);
                 frame->local = local;
                 local->inode = inode_ref (loc->inode);
                 loc_copy (&local->loc, loc);
-                local->xattr = dict_copy_with_ref (params, NULL);
+                local->xattr = dict_copy_with_ref (xdata, NULL);
                 local->mode = mode;
+                local->umask = umask;
                 local->rdev = rdev;
 
                 /* Everytime in stripe lookup, all child nodes should
                    be looked up */
                 local->call_count = priv->child_count;
 
-                /* Send a setxattr request to nodes where the
-                   files are created */
-                sprintf (size_key,
-                         "trusted.%s.stripe-size", this->name);
-                sprintf (count_key,
-                         "trusted.%s.stripe-count", this->name);
-                sprintf (index_key,
-                         "trusted.%s.stripe-index", this->name);
-
                 if (priv->xattr_supported) {
                         dict = dict_new ();
                         if (!dict) {
@@ -1914,22 +1719,22 @@ stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                         }
                         need_unref = 1;
 
-                        dict_copy (params, dict);
+                        dict_copy (xdata, dict);
 
                         ret = stripe_xattr_request_build (this, dict,
                                                           local->stripe_size,
                                                           priv->child_count,
-                                                          i);
+                                                          i, priv->coalesce);
                         if (ret)
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "failed to build xattr request");
                 } else {
-                        dict = params;
+                        dict = xdata;
                 }
 
                 STACK_WIND (frame, stripe_mknod_first_ifreg_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                            loc, mode, rdev, dict);
+                            loc, mode, rdev, umask, dict);
 
                         if (dict && need_unref)
                                 dict_unref (dict);
@@ -1938,11 +1743,11 @@ stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 
         STACK_WIND (frame, stripe_single_mknod_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, params);
+                    loc, mode, rdev, umask, xdata);
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1951,7 +1756,7 @@ int32_t
 stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t  *local   = NULL;
@@ -2008,7 +1813,7 @@ stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STRIPE_STACK_UNWIND (mkdir, frame, local->op_ret,
                                      local->op_errno, local->inode,
                                      &local->stbuf, &local->preparent,
-                                     &local->postparent);
+                                     &local->postparent, NULL);
         }
 out:
         return 0;
@@ -2017,9 +1822,9 @@ out:
 
 int32_t
 stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t  *local   = NULL;
         call_frame_t    *prev = NULL;
@@ -2050,7 +1855,7 @@ stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->stbuf      = *buf;
         local->postparent = *postparent;
         local->preparent  = *preparent;
-         
+
         local->stbuf_blocks      += buf->ia_blocks;
         local->preparent_blocks  += preparent->ia_blocks;
         local->postparent_blocks += postparent->ia_blocks;
@@ -2062,13 +1867,13 @@ stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         while (trav) {
                 STACK_WIND (frame, stripe_mkdir_cbk, trav->xlator,
                             trav->xlator->fops->mkdir, &local->loc, local->mode,
-                            local->dict);
+                            local->umask, local->xdata);
                 trav = trav->next;
         }
         return 0;
 out:
         STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                      NULL);
+                             NULL, NULL);
 
         return 0;
 
@@ -2077,7 +1882,7 @@ out:
 
 int
 stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dict_t *params)
+              mode_t umask, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
@@ -2106,18 +1911,20 @@ stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         }
         local->op_ret = -1;
         local->call_count = priv->child_count;
-        local->dict = dict_ref (params);
-        local->mode = mode;
+        if (xdata)
+                local->xdata = dict_ref (xdata);
+        local->mode  = mode;
+        local->umask = umask;
         loc_copy (&local->loc, loc);
         frame->local = local;
 
         /* Everytime in stripe lookup, all child nodes should be looked up */
         STACK_WIND (frame, stripe_first_mkdir_cbk, trav->xlator,
-                    trav->xlator->fops->mkdir, loc, mode, params);
+                    trav->xlator->fops->mkdir, loc, mode, umask, xdata);
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2126,11 +1933,12 @@ int32_t
 stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t  *local   = NULL;
         call_frame_t    *prev = NULL;
+	stripe_fd_ctx_t *fctx = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -2157,6 +1965,16 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (op_ret >= 0) {
                         local->op_ret = 0;
 
+			if (IA_ISREG(inode->ia_type)) {
+				inode_ctx_get(inode, this, (uint64_t *) &fctx);
+				if (!fctx) {
+					gf_log(this->name, GF_LOG_ERROR,
+						"failed to get stripe context");
+					op_ret = -1;
+					op_errno = EINVAL;
+				}
+			}
+
                         if (FIRST_CHILD(this) == prev->this) {
                                 local->inode      = inode_ref (inode);
                                 local->stbuf      = *buf;
@@ -2167,6 +1985,8 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->preparent_size < preparent->ia_size)
@@ -2192,14 +2012,14 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STRIPE_STACK_UNWIND (link, frame, local->op_ret,
                                      local->op_errno, local->inode,
                                      &local->stbuf, &local->preparent,
-                                     &local->postparent);
+                                     &local->postparent, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -2236,13 +2056,13 @@ stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
         while (trav) {
                 STACK_WIND (frame, stripe_link_cbk,
                             trav->xlator, trav->xlator->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -2250,7 +2070,7 @@ int32_t
 stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this, int32_t op_ret,
                                int32_t op_errno, struct iatt *preparent,
-                               struct iatt *postparent)
+                               struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2271,7 +2091,7 @@ stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
         if (!callcnt) {
                 STRIPE_STACK_UNWIND (create, frame, local->op_ret, local->op_errno,
                                      local->fd, local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -2282,12 +2102,11 @@ int32_t
 stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd,
                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
         call_frame_t     *prev = NULL;
         xlator_list_t    *trav = NULL;
 
@@ -2313,12 +2132,21 @@ stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 if (op_ret >= 0) {
+			if (IA_ISREG(buf->ia_type)) {
+				if (stripe_ctx_handle(this, prev, local, xdata))
+					gf_log(this->name, GF_LOG_ERROR,
+						"Error getting fctx info from "
+						"dict");
+			}
+
                         local->op_ret = op_ret;
 
                         local->stbuf_blocks += buf->ia_blocks;
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->preparent_size < preparent->ia_size)
@@ -2341,7 +2169,7 @@ stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                             stripe_create_fail_unlink_cbk,
                                             trav->xlator,
                                             trav->xlator->fops->unlink,
-                                            &local->loc);
+                                            &local->loc, 0, NULL);
                                 trav = trav->next;
                         }
 
@@ -2356,29 +2184,19 @@ stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->stbuf.ia_size        = local->stbuf_size;
                         local->stbuf.ia_blocks      = local->stbuf_blocks;
 
-                        fctx = GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                          gf_stripe_mt_stripe_fd_ctx_t);
-                        if (!fctx) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unwind;
-                        }
-
-                        fctx->stripe_size  = local->stripe_size;
-                        fctx->stripe_count = priv->child_count;
-                        fctx->static_array = 1;
-                        fctx->xl_array = priv->xl_array;
-                        inode_ctx_put (local->inode, this,
-                                    (uint64_t)(long)fctx);
+			stripe_copy_xl_array(local->fctx->xl_array,
+				             priv->xl_array,
+					     local->fctx->stripe_count);
+			inode_ctx_put(local->inode, this,
+					(uint64_t) local->fctx);
                 }
 
-        unwind:
                 /* Create itself has failed.. so return
                    without setxattring */
                 STRIPE_STACK_UNWIND (create, frame, local->op_ret,
                                      local->op_errno, local->fd,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 
 out:
@@ -2391,7 +2209,7 @@ int32_t
 stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd,
                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -2448,7 +2266,7 @@ stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->call_count = 1;
                 STACK_WIND (frame, stripe_create_fail_unlink_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                            &local->loc);
+                            &local->loc, 0, NULL);
                 return 0;
         }
 
@@ -2478,17 +2296,17 @@ stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         ret = stripe_xattr_request_build (this, dict,
                                                           local->stripe_size,
                                                           priv->child_count,
-                                                          i);
+                                                          i, priv->coalesce);
                         if (ret)
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "failed to build xattr request");
                 } else {
                         dict = local->xattr;
                 }
- 
+
                 STACK_WIND (frame, stripe_create_cbk, trav->xlator,
                             trav->xlator->fops->create, &local->loc,
-                            local->flags, local->mode, local->fd,
+                            local->flags, local->mode, local->umask, local->fd,
                             dict);
                 trav = trav->next;
                 if (need_unref && dict)
@@ -2510,7 +2328,7 @@ out:
  */
 int32_t
 stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+               int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
@@ -2546,16 +2364,16 @@ stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
         local->op_ret = -1;
         local->op_errno = ENOTCONN;
-        local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                     priv->pattern,
-                                                     priv->block_size);
+        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
         frame->local = local;
         local->inode = inode_ref (loc->inode);
         loc_copy (&local->loc, loc);
         local->fd = fd_ref (fd);
         local->flags = flags;
         local->mode = mode;
-        local->xattr = dict_copy_with_ref (params, NULL);
+        local->umask = umask;
+        if (xdata)
+                local->xattr = dict_ref (xdata);
 
         local->call_count = priv->child_count;
         /* Send a setxattr request to nodes where the
@@ -2569,23 +2387,23 @@ stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 }
                 need_unref = 1;
 
-                dict_copy (params, dict);
+                dict_copy (xdata, dict);
 
                 ret = stripe_xattr_request_build (this, dict,
                                                   local->stripe_size,
                                                   priv->child_count,
-                                                  i);
+                                                  i, priv->coalesce);
                 if (ret)
                         gf_log (this->name, GF_LOG_ERROR,
                                 "failed to build xattr request");
         } else {
-                        dict = params;
+                        dict = xdata;
         }
 
 
         STACK_WIND (frame, stripe_first_create_cbk, FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->create, loc, flags, mode,
-                    fd, dict);
+                    umask, fd, dict);
 
         if (need_unref && dict)
                 dict_unref (dict);
@@ -2594,13 +2412,13 @@ stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 err:
         STRIPE_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+                             NULL, NULL, xdata);
         return 0;
 }
 
 int32_t
 stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2639,7 +2457,7 @@ stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = -1;
 
                 STRIPE_STACK_UNWIND (open, frame, local->op_ret,
-                                     local->op_errno, local->fd);
+                                     local->op_errno, local->fd, xdata);
         }
 out:
         return 0;
@@ -2651,7 +2469,7 @@ out:
  */
 int32_t
 stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, int32_t wbflags)
+             int32_t flags, fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -2689,27 +2507,25 @@ stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
         /* Striped files */
         local->flags = flags;
         local->call_count = priv->child_count;
-        local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                     priv->pattern,
-                                                     priv->block_size);
+        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
 
         while (trav) {
                 STACK_WIND (frame, stripe_open_cbk, trav->xlator,
                             trav->xlator->fops->open,
                             &local->loc, local->flags, local->fd,
-                            wbflags);
+                            xdata);
                 trav = trav->next;
         }
         return 0;
 err:
-        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2742,7 +2558,7 @@ stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (!callcnt) {
                 STRIPE_STACK_UNWIND (opendir, frame, local->op_ret,
-                                     local->op_errno, local->fd);
+                                     local->op_errno, local->fd, NULL);
         }
 out:
         return 0;
@@ -2750,7 +2566,7 @@ out:
 
 
 int32_t
-stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -2783,19 +2599,19 @@ stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 
         while (trav) {
                 STACK_WIND (frame, stripe_opendir_cbk, trav->xlator,
-                            trav->xlator->fops->opendir, loc, fd);
+                            trav->xlator->fops->opendir, loc, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int32_t
 stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2835,7 +2651,7 @@ stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->failed)
                         local->op_ret = -1;
                 STRIPE_STACK_UNWIND (lk, frame, local->op_ret,
-                                     local->op_errno, &local->lock);
+                                     local->op_errno, &local->lock, NULL);
         }
 out:
         return 0;
@@ -2843,7 +2659,7 @@ out:
 
 int32_t
 stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock)
+           struct gf_flock *lock, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
@@ -2870,20 +2686,20 @@ stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
 
         while (trav) {
                 STACK_WIND (frame, stripe_lk_cbk, trav->xlator,
-                            trav->xlator->fops->lk, fd, cmd, lock);
+                            trav->xlator->fops->lk, fd, cmd, lock, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -2920,14 +2736,14 @@ stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = -1;
 
                 STRIPE_STACK_UNWIND (flush, frame, local->op_ret,
-                                     local->op_errno);
+                                     local->op_errno, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -2958,13 +2774,13 @@ stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
         while (trav) {
                 STACK_WIND (frame, stripe_flush_cbk,  trav->xlator,
-                            trav->xlator->fops->flush, fd);
+                            trav->xlator->fops->flush, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno);
+        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -2973,7 +2789,7 @@ err:
 int32_t
 stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -3009,6 +2825,9 @@ stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += prebuf->ia_blocks;
                         local->postbuf_blocks += postbuf->ia_blocks;
 
+			correct_file_size(prebuf, local->fctx, prev);
+			correct_file_size(postbuf, local->fctx, prev);
+
                         if (local->prebuf_size < prebuf->ia_size)
                                 local->prebuf_size = prebuf->ia_size;
 
@@ -3031,18 +2850,19 @@ stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STRIPE_STACK_UNWIND (fsync, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -3059,25 +2879,33 @@ stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
                 op_errno = ENOMEM;
                 goto err;
         }
-        local->op_ret = -1;
+
         frame->local = local;
+
+	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		op_errno = EINVAL;
+		goto err;
+	}
+	local->fctx = fctx;
+        local->op_ret = -1;
         local->call_count = priv->child_count;
 
         while (trav) {
                 STACK_WIND (frame, stripe_fsync_cbk, trav->xlator,
-                            trav->xlator->fops->fsync, fd, flags);
+                            trav->xlator->fops->fsync, fd, flags, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 int32_t
 stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -3112,6 +2940,9 @@ stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->stbuf = *buf;
 
                         local->stbuf_blocks += buf->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                 }
@@ -3128,7 +2959,7 @@ stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 STRIPE_STACK_UNWIND (fstat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf);
+                                     local->op_errno, &local->stbuf, NULL);
         }
 
 out:
@@ -3138,11 +2969,12 @@ out:
 int32_t
 stripe_fstat (call_frame_t *frame,
               xlator_t *this,
-              fd_t *fd)
+              fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -3163,26 +2995,35 @@ stripe_fstat (call_frame_t *frame,
         frame->local = local;
         local->call_count = priv->child_count;
 
+	if (IA_ISREG(fd->inode->ia_type)) {
+		inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         while (trav) {
                 STACK_WIND (frame, stripe_fstat_cbk, trav->xlator,
-                            trav->xlator->fops->fstat, fd);
+                            trav->xlator->fops->fstat, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
-stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
+	stripe_fd_ctx_t  *fctx = NULL;
+	int		  i, eof_idx;
+	off_t		  dest_offset, tmp_offset;
+        int32_t		  op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3190,7 +3031,6 @@ stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         VALIDATE_OR_GOTO (fd->inode, err);
 
         priv = this->private;
-        trav = this->children;
 
         /* Initialization */
         local = mem_get0 (this->local_pool);
@@ -3202,22 +3042,60 @@ stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         frame->local = local;
         local->call_count = priv->child_count;
 
-        while (trav) {
-                STACK_WIND (frame, stripe_truncate_cbk, trav->xlator,
-                            trav->xlator->fops->ftruncate, fd, offset);
-                trav = trav->next;
-        }
+	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
+		op_errno = EINVAL;
+		goto err;
+	}
+	if (!fctx->stripe_count) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe count");
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	local->fctx = fctx;
+	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (!fctx->xl_array[i]) {
+			gf_log(this->name, GF_LOG_ERROR, "no xlator at index "
+				"%d", i);
+			op_errno = EINVAL;
+			goto err;
+		}
+
+		if (fctx->stripe_coalesce) {
+			if (i < eof_idx)
+				tmp_offset = roof(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else if (i > eof_idx)
+				tmp_offset = floor(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else
+				tmp_offset = offset;
+
+			dest_offset = coalesced_offset(tmp_offset,
+				fctx->stripe_size, fctx->stripe_count);
+		} else {
+			dest_offset = offset;
+		}
+
+		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
+			fctx->xl_array[i]->fops->ftruncate, fd, dest_offset,
+			NULL);
+	}
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -3254,14 +3132,14 @@ stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = -1;
 
                 STRIPE_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                     local->op_errno);
+                                     local->op_errno, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -3288,20 +3166,20 @@ stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
 
         while (trav) {
                 STACK_WIND (frame, stripe_fsyncdir_cbk, trav->xlator,
-                            trav->xlator->fops->fsyncdir, fd, flags);
+                            trav->xlator->fops->fsyncdir, fd, flags, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         i = 0;
         int32_t         callcnt = 0;
@@ -3311,6 +3189,7 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct iatt     tmp_stbuf = {0,};
         struct iobref  *tmp_iobref = NULL;
         struct iobuf   *iobuf = NULL;
+	call_frame_t   *prev = NULL;
 
         if (!this || !frame || !frame->local) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3318,13 +3197,16 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         local = frame->local;
+	prev = cookie;
 
         LOCK (&frame->lock);
         {
                 callcnt = --local->call_count;
-                if (op_ret != -1)
+                if (op_ret != -1) {
+			correct_file_size(buf, local->fctx, prev);
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
+		}
         }
         UNLOCK (&frame->lock);
 
@@ -3363,9 +3245,11 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         goto done;
                                 }
                                 memset (iobuf->ptr, 0, vec[count].iov_len);
-                                iobref_add (local->iobref, iobuf);
                                 vec[count].iov_base = iobuf->ptr;
 
+                                iobref_add (local->iobref, iobuf);
+                                iobuf_unref(iobuf);
+
                                 op_ret += vec[count].iov_len;
                                 count++;
                         }
@@ -3383,11 +3267,10 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 GF_FREE (local->replies);
                 tmp_iobref = local->iobref;
                 STRIPE_STACK_UNWIND (readv, frame, op_ret, op_errno, vec,
-                                     count, &tmp_stbuf, tmp_iobref);
+                                     count, &tmp_stbuf, tmp_iobref, NULL);
 
                 iobref_unref (tmp_iobref);
-                if (vec)
-                        GF_FREE (vec);
+                GF_FREE (vec);
         }
 out:
         return 0;
@@ -3400,7 +3283,7 @@ out:
 int32_t
 stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
         int32_t         index = 0;
         int32_t         callcnt = 0;
@@ -3414,6 +3297,7 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct iatt    *tmp_stbuf_p = NULL; //need it for a warning
         struct iobref  *tmp_iobref = NULL;
         stripe_fd_ctx_t  *fctx = NULL;
+	call_frame_t	*prev = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3422,6 +3306,7 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local  = frame->local;
         index  = local->node_index;
+	prev = cookie;
         mframe = local->orig_frame;
         if (!mframe)
                 goto out;
@@ -3441,6 +3326,9 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         mlocal->replies[index].stbuf  = *stbuf;
                         mlocal->replies[index].count  = count;
                         mlocal->replies[index].vector = iov_dup (vector, count);
+
+			correct_file_size(stbuf, fctx, prev);
+
                         if (local->stbuf_size < stbuf->ia_size)
                                 local->stbuf_size = stbuf->ia_size;
                         local->stbuf_blocks += stbuf->ia_blocks;
@@ -3512,11 +3400,10 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 tmp_stbuf_p = &tmp_stbuf;
                 WIPE (tmp_stbuf_p);
                 STRIPE_STACK_UNWIND (readv, mframe, op_ret, op_errno, final_vec,
-                                     final_count, &tmp_stbuf, tmp_iobref);
+                                     final_count, &tmp_stbuf, tmp_iobref, NULL);
 
                 iobref_unref (tmp_iobref);
-                if (final_vec)
-                        GF_FREE (final_vec);
+                GF_FREE (final_vec);
         }
 
         goto out;
@@ -3528,7 +3415,7 @@ check_size:
                 STACK_WIND (mframe, stripe_readv_fstat_cbk,
                             (fctx->xl_array[index]),
                             (fctx->xl_array[index])->fops->fstat,
-                            mlocal->fd);
+                            mlocal->fd, NULL);
         }
 
 out:
@@ -3540,7 +3427,7 @@ end:
 
 int32_t
 stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              size_t size, off_t offset, uint32_t flags)
+              size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         int32_t           op_errno = EINVAL;
         int32_t           idx = 0;
@@ -3553,6 +3440,7 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         uint64_t          stripe_size = 0;
         off_t             rounded_start = 0;
         off_t             frame_offset = offset;
+	off_t		  dest_offset = 0;
         stripe_local_t   *local = NULL;
         call_frame_t     *rframe = NULL;
         stripe_local_t   *rlocal = NULL;
@@ -3571,6 +3459,8 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
         stripe_size = fctx->stripe_size;
 
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
         if (!stripe_size) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Wrong stripe size for the file");
@@ -3593,8 +3483,8 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         frame->local = local;
 
         /* This is where all the vectors should be copied. */
-        local->replies = GF_CALLOC (num_stripe, sizeof (struct readv_replies),
-                                    gf_stripe_mt_readv_replies);
+        local->replies = GF_CALLOC (num_stripe, sizeof (struct stripe_replies),
+                                    gf_stripe_mt_stripe_replies);
         if (!local->replies) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3623,9 +3513,16 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 rlocal->readv_size = frame_size;
                 rframe->local = rlocal;
                 idx = (index % fctx->stripe_count);
+
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(frame_offset,
+				stripe_size, fctx->stripe_count);
+		else
+			dest_offset = frame_offset;
+
                 STACK_WIND (rframe, stripe_readv_cbk, fctx->xl_array[idx],
                             fctx->xl_array[idx]->fops->readv,
-                            fd, frame_size, frame_offset, flags);
+                            fd, frame_size, dest_offset, flags, xdata);
 
                 frame_offset += frame_size;
         }
@@ -3635,7 +3532,7 @@ err:
         if (rframe)
                 STRIPE_STACK_DESTROY (rframe);
 
-        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
         return 0;
 }
 
@@ -3643,11 +3540,15 @@ err:
 int32_t
 stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
         call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+	struct stripe_replies *reply = NULL;
+	int32_t		i = 0;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3656,39 +3557,82 @@ stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         prev  = cookie;
         local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
 
         LOCK(&frame->lock);
         {
-                callcnt = ++local->call_count;
+                callcnt = ++mlocal->call_count;
+
+		mlocal->replies[local->node_index].op_ret = op_ret;
+		mlocal->replies[local->node_index].op_errno = op_errno;
 
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        local->op_ret = -1;
-                }
                 if (op_ret >= 0) {
-                        local->op_ret += op_ret;
-                        local->post_buf = *postbuf;
-                        local->pre_buf = *prebuf;
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
                 }
         }
         UNLOCK (&frame->lock);
 
-        if ((callcnt == local->wind_count) && local->unwind) {
-                STRIPE_STACK_UNWIND (writev, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+		/*
+		 * Only return the number of consecutively written bytes up until
+		 * the first error. Only return an error if it occurs first.
+		 *
+		 * When a short write occurs, the application should retry at the
+		 * appropriate offset, at which point we'll potentially pass back
+		 * the error.
+		 */
+		for (i = 0, reply = mlocal->replies; i < mlocal->wind_count;
+			i++, reply++) {
+			if (reply->op_ret == -1) {
+				gf_log(this->name, GF_LOG_DEBUG, "reply %d "
+					"returned error %s", i,
+					strerror(reply->op_errno));
+				if (!mlocal->op_ret) {
+					mlocal->op_ret = -1;
+					mlocal->op_errno = reply->op_errno;
+				}
+				break;
+			}
+
+			mlocal->op_ret += reply->op_ret;
+
+			if (reply->op_ret < reply->requested_size)
+				break;
+		}
+
+		GF_FREE(mlocal->replies);
+
+                STRIPE_STACK_UNWIND (writev, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
         }
 out:
+	STRIPE_STACK_DESTROY(frame);
         return 0;
 }
 
 int32_t
 stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                struct iovec *vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref *iobref)
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         struct iovec     *tmp_vec = NULL;
         stripe_local_t   *local = NULL;
@@ -3702,6 +3646,12 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         off_t             fill_size = 0;
         uint64_t          stripe_size = 0;
         uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	off_t		  rounded_start = 0;
+	off_t		  rounded_end = 0;
+	int32_t		  total_chunks = 0;
+	call_frame_t	  *wframe = NULL;
+	stripe_local_t	  *wlocal = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3716,6 +3666,8 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
         stripe_size = fctx->stripe_size;
 
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
         /* File has to be stripped across the child nodes */
         for (idx = 0; idx< count; idx ++) {
                 total_size += vector[idx].iov_len;
@@ -3729,6 +3681,7 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         }
         frame->local = local;
         local->stripe_size = stripe_size;
+	local->fctx = fctx;
 
         if (!stripe_size) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -3737,7 +3690,27 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto err;
         }
 
+	rounded_start = floor(offset, stripe_size);
+	rounded_end = roof(offset + total_size, stripe_size);
+	total_chunks = (rounded_end - rounded_start) / stripe_size;
+	local->replies = GF_CALLOC(total_chunks, sizeof(struct stripe_replies),
+				gf_stripe_mt_stripe_replies);
+	if (!local->replies) {
+		op_errno = ENOMEM;
+		goto err;
+	}
+
+	total_chunks = 0;
         while (1) {
+		wframe = copy_frame(frame);
+		wlocal = mem_get0(this->local_pool);
+		if (!wlocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		wlocal->orig_frame = frame;
+		wframe->local = wlocal;
+
                 /* Send striped chunk of the vector to child
                    nodes appropriately. */
                 idx = (((offset + offset_offset) /
@@ -3765,21 +3738,537 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 if (remaining_size == 0)
                         local->unwind = 1;
 
-                STACK_WIND (frame, stripe_writev_cbk, fctx->xl_array[idx],
+		/*
+		 * Store off the request index (with respect to the chunk of the
+		 * initial offset) and the size of the request. This is required
+		 * in the callback to calculate an appropriate return value in
+		 * the event of a write failure in one or more requests.
+		 */
+		wlocal->node_index = total_chunks;
+		local->replies[total_chunks].requested_size = fill_size;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+                STACK_WIND (wframe, stripe_writev_cbk, fctx->xl_array[idx],
                             fctx->xl_array[idx]->fops->writev, fd, tmp_vec,
-                            tmp_count, offset + offset_offset, flags, iobref);
+                            tmp_count, dest_offset, flags, iobref,
+                            xdata);
+
                 GF_FREE (tmp_vec);
                 offset_offset += fill_size;
+		total_chunks++;
                 if (remaining_size == 0)
                         break;
         }
 
         return 0;
 err:
-        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+	if (wframe)
+		STRIPE_STACK_DESTROY(wframe);
+
+        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+stripe_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (fallocate, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send fallocate request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single fallocate per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_fallocate_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->fallocate, fd, mode,
+			   dest_offset, fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+stripe_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (discard, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
         return 0;
 }
 
+int32_t
+stripe_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send discard request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single discard per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_discard_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->discard, fd, dest_offset,
+			   fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt    = 0;
+        stripe_local_t *local      = NULL;
+        stripe_local_t *mlocal     = NULL;
+        call_frame_t   *prev       = NULL;
+        call_frame_t   *mframe     = NULL;
+
+        GF_ASSERT (frame);
+
+        if (!this || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+        mframe = local->orig_frame;
+        mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
+                        mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+                        correct_file_size(prebuf, mlocal->fctx, prev);
+                        correct_file_size(postbuf, mlocal->fctx, prev);
+
+                        if (mlocal->prebuf_size < prebuf->ia_size)
+                                mlocal->prebuf_size = prebuf->ia_size;
+                        if (mlocal->postbuf_size < postbuf->ia_size)
+                                mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+                /* return the first failure */
+                if (mlocal->op_ret == 0) {
+                        mlocal->op_ret = op_ret;
+                        mlocal->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+                mlocal->post_buf.ia_size = mlocal->postbuf_size;
+                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+        STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
+{
+        stripe_local_t   *local            = NULL;
+        stripe_fd_ctx_t  *fctx             = NULL;
+        int32_t           op_errno         = 1;
+        int32_t           idx              = 0;
+        int32_t           offset_offset    = 0;
+        int32_t           remaining_size   = 0;
+        off_t             fill_size        = 0;
+        uint64_t          stripe_size      = 0;
+        uint64_t          tmp_fctx         = 0;
+        off_t             dest_offset      = 0;
+        call_frame_t     *fframe           = NULL;
+        stripe_local_t   *flocal           = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+        local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+                fframe = copy_frame(frame);
+                flocal = mem_get0(this->local_pool);
+                if (!flocal) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                flocal->orig_frame = frame;
+                fframe->local = flocal;
+
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+                dest_offset = offset + offset_offset;
+                if (fctx->stripe_coalesce)
+                        dest_offset = coalesced_offset(dest_offset,
+                                                       local->stripe_size,
+                                                       fctx->stripe_count);
+
+                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
+                           fctx->xl_array[idx]->fops->zerofill, fd,
+                           dest_offset, fill_size, xdata);
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+        if (fframe)
+                STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
 int32_t
 stripe_release (xlator_t *this, fd_t *fd)
@@ -3817,6 +4306,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         stripe_private_t *priv = NULL;
         int               down_client = 0;
         int               i = 0;
+        gf_boolean_t      heard_from_all_children = _gf_false;
 
         if (!this)
                 return 0;
@@ -3828,30 +4318,34 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         switch (event)
         {
         case GF_EVENT_CHILD_UP:
-        case GF_EVENT_CHILD_CONNECTING:
         {
                 /* get an index number to set */
                 for (i = 0; i < priv->child_count; i++) {
                         if (data == priv->xl_array[i])
                                 break;
                 }
-                priv->state[i] = 1;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!priv->state[i])
-                                down_client++;
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_UP bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
                 }
 
                 LOCK (&priv->lock);
                 {
-                        priv->nodes_down = down_client;
                         if (data == FIRST_CHILD (this))
                                 priv->first_child_down = 0;
-                        if (!priv->nodes_down)
-                                default_notify (this, event, data);
+                        priv->last_event[i] = event;
                 }
                 UNLOCK (&priv->lock);
         }
         break;
+        case GF_EVENT_CHILD_CONNECTING:
+        {
+                // 'CONNECTING' doesn't ensure its CHILD_UP, so do nothing
+                goto out;
+        }
         case GF_EVENT_CHILD_DOWN:
         {
                 /* get an index number to set */
@@ -3859,20 +4353,19 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                         if (data == priv->xl_array[i])
                                 break;
                 }
-                priv->state[i] = 0;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!priv->state[i])
-                                down_client++;
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
                 }
 
                 LOCK (&priv->lock);
                 {
-                        priv->nodes_down = down_client;
-
                         if (data == FIRST_CHILD (this))
                                 priv->first_child_down = 1;
-                        if (priv->nodes_down)
-                                default_notify (this, event, data);
+                        priv->last_event[i] = event;
                 }
                 UNLOCK (&priv->lock);
         }
@@ -3882,177 +4375,322 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         {
                 /* */
                 default_notify (this, event, data);
+                goto out;
         }
         break;
         }
 
+        // Consider child as down if it's last_event is not CHILD_UP
+        for (i = 0, down_client = 0; i < priv->child_count; i++)
+                if (priv->last_event[i] != GF_EVENT_CHILD_UP)
+                        down_client++;
+
+        LOCK (&priv->lock);
+        {
+                priv->nodes_down = down_client;
+        }
+        UNLOCK (&priv->lock);
+
+        heard_from_all_children = _gf_true;
+        for (i = 0; i < priv->child_count; i++)
+                if (!priv->last_event[i])
+                        heard_from_all_children = _gf_false;
+
+        if (heard_from_all_children)
+                default_notify (this, event, data);
+out:
         return 0;
 }
 
 int
-set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
+stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
+                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *stripe_str = NULL;
-        char                  *pattern = NULL;
-        char                  *num = NULL;
-        struct stripe_options *temp_stripeopt = NULL;
-        struct stripe_options *stripe_opt = NULL;
-
-        if (!this || !priv || !data)
-                goto out;
+        int ret = -1;
+        int call_cnt = 0;
+        stripe_local_t *local = NULL;
 
-        /* Get the pattern for striping.
-           "option block-size *avi:10MB" etc */
-        stripe_str = strtok_r (data, ",", &tmp_str);
-        while (stripe_str) {
-                dup_str = gf_strdup (stripe_str);
-                stripe_opt = CALLOC (1, sizeof (struct stripe_options));
-                if (!stripe_opt) {
-                        GF_FREE (dup_str);
-                        goto out;
-                }
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
+                return ret;
+        }
 
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!num) {
-                        num = pattern;
-                        pattern = "*";
-                }
-                if (gf_string2bytesize (num, &stripe_opt->block_size) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid number format \"%s\"", num);
-                        goto out;
-                }
+        local = frame->local;
 
-                if (stripe_opt->block_size < 512) {
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
-                                "%s. Should be atleast 512 bytes", num);
-                        goto out;
-                }
-                if (stripe_opt->block_size % 512) {
-                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
-                                " be a multiple of 512 bytes", num);
-                        goto out;
-                }
+        LOCK (&frame->lock);
+        {
+                call_cnt = --local->wind_count;
 
-                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
+                /**
+                 * We overwrite ->op_* values here for subsequent faliure
+                 * conditions, hence we propogate the last errno down the
+                 * stack.
+                 */
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                        goto unlock;
+                }
+        }
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "block-size : pattern %s : size %"PRId64,
-                        stripe_opt->path_pattern, stripe_opt->block_size);
+ unlock:
+        UNLOCK (&frame->lock);
 
-                if (!priv->pattern) {
-                        priv->pattern = stripe_opt;
-                } else {
-                        temp_stripeopt = priv->pattern;
-                        while (temp_stripeopt->next)
-                                temp_stripeopt = temp_stripeopt->next;
-                        temp_stripeopt->next = stripe_opt;
-                }
-                stripe_str = strtok_r (NULL, ",", &tmp_str);
-                GF_FREE (dup_str);
+        if (!call_cnt) {
+                STRIPE_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                     local->op_errno, xdata);
         }
 
-        ret = 0;
-out:
-        return ret;
+        return 0;
 }
 
-int32_t
-stripe_iatt_merge (struct iatt *from, struct iatt *to)
+#ifdef HAVE_BD_XLATOR
+int
+stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
 {
-        if (to->ia_size < from->ia_size)
-                to->ia_size = from->ia_size;
-        if (to->ia_mtime < from->ia_mtime)
-                to->ia_mtime = from->ia_mtime;
-        if (to->ia_ctime < from->ia_ctime)
-                to->ia_ctime = from->ia_ctime;
-        if (to->ia_atime < from->ia_atime)
-                to->ia_atime = from->ia_atime;
+        gf_boolean_t *is_bd = data;
+
+        if (data == NULL)
+                return 0;
+
+        if (XATTR_IS_BD (key))
+            *is_bd = _gf_true;
+
         return 0;
 }
 
-int
-stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno)
+inline gf_boolean_t
+stripe_setxattr_is_bd (dict_t *dict)
 {
-        STRIPE_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
-        return 0;
+        gf_boolean_t is_bd = _gf_false;
+
+        if (dict == NULL)
+                goto out;
+
+        dict_foreach (dict, stripe_is_bd, &is_bd);
+out:
+        return is_bd;
 }
+#else
+#define stripe_setxattr_is_bd(dict) _gf_false
+#endif
 
 int
 stripe_setxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *dict, int flags)
+                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
-        int32_t         op_errno = EINVAL;
+        int32_t           op_errno = EINVAL;
+        xlator_list_t    *trav     = NULL;
+        stripe_private_t *priv     = NULL;
+        stripe_local_t   *local    = NULL;
+        int               i        = 0;
+        gf_boolean_t      is_bd    = _gf_false;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   trav, op_errno, err);
+                                   op_errno, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        frame->local = local;
+        local->wind_count = priv->child_count;
+        local->op_ret = local->op_errno = 0;
+
+        is_bd = stripe_setxattr_is_bd (dict);
+
+        /**
+         * Set xattrs for directories on all subvolumes. Additionally
+         * this power is only given to a special client. Bd xlator
+         * also needs xattrs for regular files (ie LVs)
+         */
+        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
+             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
+                for (i = 0; i < priv->child_count; i++, trav = trav->next) {
+                        STACK_WIND (frame, stripe_setxattr_cbk,
+                                    trav->xlator, trav->xlator->fops->setxattr,
+                                    loc, dict, flags, xdata);
+                }
+        } else {
+                local->wind_count = 1;
+                STACK_WIND (frame, stripe_setxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            loc, dict, flags, xdata);
+        }
 
-        STACK_WIND (frame, stripe_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
         return 0;
 err:
-        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno);
+        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno, NULL);
         return 0;
 }
 
 
 int
 stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno)
+                      xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+{
+        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+stripe_is_special_key (dict_t *this,
+                       char *key,
+                       data_t *value,
+                       void *data)
+{
+        gf_boolean_t *is_special = NULL;
+
+        if (data == NULL) {
+                goto out;
+        }
+
+        is_special = data;
+
+        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
+                *is_special = _gf_true;
+
+out:
+        return 0;
+}
+
+int32_t
+stripe_fsetxattr_everyone_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata)
 {
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+        int             call_count = 0;
+        stripe_local_t *local      = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                call_count = --local->wind_count;
+
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (call_count == 0) {
+                STRIPE_STACK_UNWIND (fsetxattr, frame, local->op_ret,
+                                     local->op_errno, NULL);
+        }
         return 0;
 }
 
 int
+stripe_fsetxattr_to_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              dict_t *dict, int flags, dict_t *xdata)
+{
+        xlator_list_t    *trav  = NULL;
+        stripe_private_t *priv  = NULL;
+        int               ret   = -1;
+        stripe_local_t   *local = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (local == NULL) {
+                goto out;
+        }
+
+        frame->local = local;
+
+        local->wind_count = priv->child_count;
+
+        trav = this->children;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_fsetxattr_everyone_cbk,
+                            trav->xlator, trav->xlator->fops->fsetxattr,
+                            fd, dict, flags, xdata);
+                trav = trav->next;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+inline gf_boolean_t
+stripe_fsetxattr_is_special (dict_t *dict)
+{
+        gf_boolean_t is_spl = _gf_false;
+
+        if (dict == NULL) {
+                goto out;
+        }
+
+        dict_foreach (dict, stripe_is_special_key, &is_spl);
+
+out:
+        return is_spl;
+}
+
+int
 stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int flags)
+                  dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
-        int32_t         op_ret   = -1;
-        int32_t         op_errno = EINVAL;
+        int32_t      op_ret = -1, ret = -1, op_errno = EINVAL;
+        gf_boolean_t is_spl = _gf_false;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   trav, op_errno, err);
+                                   op_errno, err);
+
+        is_spl = stripe_fsetxattr_is_special (dict);
+        if (is_spl) {
+                ret = stripe_fsetxattr_to_everyone (frame, this, fd, dict,
+                                                    flags, xdata);
+                if (ret < 0) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+
+                goto out;
+        }
 
         STACK_WIND (frame, stripe_fsetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, dict, flags, xdata);
+out:
         return 0;
- err:
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+err:
+        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
 int
-stripe_removexattr_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret, int32_t op_errno)
+stripe_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STRIPE_STACK_UNWIND (removexattr, frame, op_ret, op_errno);
+        STRIPE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 stripe_removexattr (call_frame_t *frame, xlator_t *this,
-                    loc_t *loc, const char *name)
+                    loc_t *loc, const char *name, dict_t *xdata)
 {
         int32_t         op_errno = EINVAL;
 
@@ -4067,25 +4705,25 @@ stripe_removexattr (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, stripe_removexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->removexattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 err:
-        STRIPE_STACK_UNWIND (removexattr, frame, -1,  op_errno);
+        STRIPE_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
         return 0;
 }
 
 
 int
-stripe_fremovexattr_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno)
+stripe_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno);
+        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 stripe_fremovexattr (call_frame_t *frame, xlator_t *this,
-                     fd_t *fd, const char *name)
+                     fd_t *fd, const char *name, dict_t *xdata)
 {
         int32_t         op_ret   = -1;
         int32_t         op_errno = EINVAL;
@@ -4100,10 +4738,10 @@ stripe_fremovexattr (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, stripe_fremovexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name);
+                    fd, name, xdata);
         return 0;
  err:
-        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno);
+        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -4138,8 +4776,15 @@ stripe_readdirp_lookup_cbk (call_frame_t *frame, void *cookie,
                         local->op_ret = op_ret;
                         goto unlock;
                 }
+
+		if (stripe_ctx_handle(this, prev, local, xattr))
+			gf_log(this->name, GF_LOG_ERROR,
+				"Error getting fctx info from dict.");
+
+		correct_file_size(stbuf, local->fctx, prev);
+
                 stripe_iatt_merge (stbuf, &entry->d_stat);
-                stripe_ctx_handle (this, prev, local, xattr);
+                local->stbuf_blocks += stbuf->ia_blocks;
         }
 unlock:
         UNLOCK(&frame->lock);
@@ -4158,6 +4803,7 @@ unlock:
                                 main_local->op_errno = local->op_errno;
                                 main_local->op_ret = local->op_ret;
                         }
+                        entry->d_stat.ia_blocks = local->stbuf_blocks;
                 }
                 UNLOCK (&main_frame->lock);
                 if (done) {
@@ -4165,7 +4811,7 @@ unlock:
                         STRIPE_STACK_UNWIND (readdir, main_frame,
                                              main_local->op_ret,
                                              main_local->op_errno,
-                                             &main_local->entries);
+                                             &main_local->entries, NULL);
                         gf_dirent_free (&main_local->entries);
                         stripe_local_wipe (main_local);
                         mem_put (main_local);
@@ -4181,17 +4827,15 @@ unlock:
 
 int32_t
 stripe_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *orig_entries)
+                     int32_t op_ret, int32_t op_errno,
+                     gf_dirent_t *orig_entries, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         call_frame_t   *prev = NULL;
         gf_dirent_t    *local_entry = NULL;
-        int32_t        ret = -1;
         gf_dirent_t    *tmp_entry = NULL;
         xlator_list_t  *trav = NULL;
         loc_t          loc = {0, };
-        inode_t        *inode = NULL;
-        char           *path;
         int32_t        count = 0;
         stripe_private_t *priv = NULL;
         int32_t        subvols = 0;
@@ -4235,15 +4879,14 @@ unlock:
 
         xattrs = dict_new ();
         if (xattrs)
-                (void) stripe_xattr_request_build (this, xattrs, 0, 0, 0);
+                (void) stripe_xattr_request_build (this, xattrs, 0, 0, 0, 0);
         count = op_ret;
-        ret = 0;
         list_for_each_entry_safe (local_entry, tmp_entry,
                                   (&local->entries.list), list) {
 
                 if (!local_entry)
                         break;
-                if (!IA_ISREG (local_entry->d_stat.ia_type)) {
+                if (!IA_ISREG (local_entry->d_stat.ia_type) || !local_entry->inode) {
                         LOCK (&frame->lock);
                         {
                                 local->wind_count--;
@@ -4253,28 +4896,6 @@ unlock:
                         continue;
                 }
 
-                inode = inode_new (local->fd->inode->table);
-                if (!inode)
-                        goto out;
-
-                loc.inode = inode;
-                loc.parent = local->fd->inode;
-                ret = inode_path (local->fd->inode, local_entry->d_name, &path);
-                if (ret != -1) {
-                        loc.path = path;
-                } else  if (inode) {
-                        ret = inode_path (inode, NULL, &path);
-                        if (ret != -1) {
-                                loc.path = path;
-                        } else {
-                                goto out;
-                        }
-                }
-
-                loc.name = strrchr (loc.path, '/');
-                loc.name++;
-                uuid_copy (loc.gfid, local_entry->d_stat.ia_gfid);
-
                 local_frame = copy_frame (frame);
 
                 if (!local_frame) {
@@ -4290,6 +4911,10 @@ unlock:
                         goto out;
                 }
 
+                loc.inode = inode_ref (local_entry->inode);
+
+                uuid_copy (loc.gfid, local_entry->d_stat.ia_gfid);
+
                 local_ent->orig_frame = frame;
 
                 local_ent->call_count = subvols;
@@ -4305,14 +4930,14 @@ unlock:
                                     &loc, xattrs);
                         trav = trav->next;
                 }
-                inode_unref (loc.inode);
+                loc_wipe (&loc);
         }
 out:
         if (!count) {
                 /* all entries are directories */
                 frame->local = NULL;
                 STRIPE_STACK_UNWIND (readdir, frame, local->op_ret,
-                                     local->op_errno, &local->entries);
+                                     local->op_errno, &local->entries, NULL);
                 gf_dirent_free (&local->entries);
                 stripe_local_wipe (local);
                 mem_put (local);
@@ -4324,14 +4949,13 @@ out:
 }
 int32_t
 stripe_readdirp (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, size_t size, off_t off, dict_t *dict)
+                 fd_t *fd, size_t size, off_t off, dict_t *xdata)
 {
         stripe_local_t  *local  = NULL;
         stripe_private_t *priv = NULL;
         xlator_list_t   *trav = NULL;
         int             op_errno = -1;
 
-
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
@@ -4365,15 +4989,16 @@ stripe_readdirp (call_frame_t *frame, xlator_t *this,
                 goto err;
 
         STACK_WIND (frame, stripe_readdirp_cbk, trav->xlator,
-                    trav->xlator->fops->readdirp, fd, size, off, dict);
+                    trav->xlator->fops->readdirp, fd, size, off, xdata);
         return 0;
 err:
         op_errno = (op_errno == -1) ? errno : op_errno;
-        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 
 }
+
 int32_t
 mem_acct_init (xlator_t *this)
 {
@@ -4394,21 +5019,86 @@ out:
         return ret;
 }
 
+static int
+clear_pattern_list (stripe_private_t *priv)
+{
+        struct stripe_options *prev = NULL;
+        struct stripe_options *trav = NULL;
+        int                    ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
+
+        trav = priv->pattern;
+        priv->pattern = NULL;
+        while (trav) {
+                prev = trav;
+                trav = trav->next;
+                GF_FREE (prev);
+        }
+
+        ret = 0;
+ out:
+        return ret;
+
+
+}
+
 
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
 
-	stripe_private_t *priv = NULL;
-	int		  ret = -1;
+        stripe_private_t *priv = NULL;
+        data_t           *data = NULL;
+        int               ret = -1;
+        volume_option_t  *opt = NULL;
 
-	priv = this->private;
+        GF_ASSERT (this);
+        GF_ASSERT (this->private);
+
+        priv = this->private;
 
-        GF_OPTION_RECONF ("block-size", priv->block_size, options, size, out);
 
         ret = 0;
-out:
-	return ret;
+        LOCK (&priv->lock);
+        {
+                ret = clear_pattern_list (priv);
+                if (ret)
+                         goto unlock;
+
+                data = dict_get (options, "block-size");
+                if (data) {
+                        ret = set_stripe_block_size (this, priv, data->data);
+                        if (ret)
+                                goto unlock;
+                } else {
+                        opt = xlator_volume_option_get (this, "block-size");
+                        if (!opt) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "option 'block-size' not found");
+                                ret = -1;
+                                goto unlock;
+                        }
+
+                        if (gf_string2bytesize (opt->default_value, &priv->block_size)){
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to set default block-size ");
+                                ret = -1;
+                                goto unlock;
+                        }
+                }
+
+		GF_OPTION_RECONF("coalesce", priv->coalesce, options, bool,
+				unlock);
+        }
+ unlock:
+        UNLOCK (&priv->lock);
+        if (ret)
+                goto out;
+
+        ret = 0;
+ out:
+        return ret;
 
 }
 
@@ -4421,6 +5111,7 @@ int32_t
 init (xlator_t *this)
 {
         stripe_private_t *priv = NULL;
+        volume_option_t  *opt = NULL;
         xlator_list_t    *trav = NULL;
         data_t           *data = NULL;
         int32_t           count = 0;
@@ -4464,9 +5155,9 @@ init (xlator_t *this)
         if (!priv->xl_array)
                 goto out;
 
-        priv->state = GF_CALLOC (count, sizeof (int8_t),
-                                 gf_stripe_mt_int8_t);
-        if (!priv->state)
+        priv->last_event = GF_CALLOC (count, sizeof (int),
+                                      gf_stripe_mt_int32_t);
+        if (!priv->last_event)
                 goto out;
 
         priv->child_count = count;
@@ -4486,22 +5177,41 @@ init (xlator_t *this)
                 goto out;
         }
 
-
-        GF_OPTION_INIT ("block-size", priv->block_size, size, out);
-
-        /* option stripe-pattern *avi:1GB,*pdf:4096 */
-        data = dict_get (this->options, "block-size");
-        if (data) {
-                ret = set_stripe_block_size (this, priv, data->data);
-                if (ret)
-                        goto out;
+        ret = 0;
+        LOCK (&priv->lock);
+        {
+                opt = xlator_volume_option_get (this, "block-size");
+                if (!opt) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "option 'block-size' not found");
+                        ret = -1;
+                        goto unlock;
+                }
+                if (gf_string2bytesize (opt->default_value, &priv->block_size)){
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set default block-size ");
+                        ret = -1;
+                        goto unlock;
+                }
+                /* option stripe-pattern *avi:1GB,*pdf:16K */
+                data = dict_get (this->options, "block-size");
+                if (data) {
+                        ret = set_stripe_block_size (this, priv, data->data);
+                        if (ret)
+                                goto unlock;
+                }
         }
+ unlock:
+        UNLOCK (&priv->lock);
+        if (ret)
+                goto out;
 
         GF_OPTION_INIT ("use-xattr", priv->xattr_supported, bool, out);
-
         /* notify related */
         priv->nodes_down = priv->child_count;
 
+	GF_OPTION_INIT("coalesce", priv->coalesce, bool, out);
+
         this->local_pool = mem_pool_new (stripe_local_t, 128);
         if (!this->local_pool) {
                 ret = -1;
@@ -4516,8 +5226,7 @@ init (xlator_t *this)
 out:
         if (ret) {
                 if (priv) {
-                        if (priv->xl_array)
-                                GF_FREE (priv->xl_array);
+                        GF_FREE (priv->xl_array);
                         GF_FREE (priv);
                 }
         }
@@ -4541,15 +5250,15 @@ fini (xlator_t *this)
         priv = this->private;
         if (priv) {
                 this->private = NULL;
-                if (priv->xl_array)
-                        GF_FREE (priv->xl_array);
+                GF_FREE (priv->xl_array);
 
                 trav = priv->pattern;
                 while (trav) {
                         prev = trav;
                         trav = trav->next;
-                        FREE (prev);
+                        GF_FREE (prev);
                 }
+                GF_FREE (priv->last_event);
                 LOCK_DESTROY (&priv->lock);
                 GF_FREE (priv);
         }
@@ -4560,21 +5269,23 @@ out:
 
 int32_t
 stripe_getxattr_unwind (call_frame_t *frame,
-                        int op_ret, int op_errno, dict_t *dict)
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
 {
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 int
-stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie,  xlator_t *this,
-                              int op_ret, int op_errno, dict_t *xattr)
+stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, dict_t *xattr,
+                              dict_t *xdata)
 {
 
         char        size_key[256]  = {0,};
         char        index_key[256] = {0,};
         char        count_key[256] = {0,};
+        char        coalesce_key[256] = {0,};
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (frame->local, out);
@@ -4585,13 +5296,15 @@ stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie,  xlator_t *this
         sprintf (size_key, "trusted.%s.stripe-size", this->name);
         sprintf (count_key, "trusted.%s.stripe-count", this->name);
         sprintf (index_key, "trusted.%s.stripe-index", this->name);
+	sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
 
         dict_del (xattr, size_key);
         dict_del (xattr, count_key);
         dict_del (xattr, index_key);
+        dict_del (xattr, coalesce_key);
 
 out:
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
         return 0;
 
@@ -4599,7 +5312,7 @@ out:
 
 int
 stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xattr)
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
         int                     call_cnt = 0;
         stripe_local_t         *local = NULL;
@@ -4629,139 +5342,35 @@ stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         if (!call_cnt) {
                 STRIPE_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                     local->xattr);
+                                     local->xattr, xdata);
         }
 
         return 0;
 }
 
 int32_t
-stripe_xattr_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        int32_t              len   = 0;
-        char                *sbuf  = NULL;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!buffer || !local || !local->xattr_list)
-                goto out;
-
-        sbuf = buffer;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-                len = xattr->xattr_len;
-
-                if (len && xattr && xattr->xattr_value) {
-                        memcpy (buffer, xattr->xattr_value, len);
-                        buffer += len;
-                        *buffer++ = ' ';
-                }
-        }
-
-        *--buffer = '\0';
-        if (total)
-                *total = buffer - sbuf;
-        ret = 0;
-
- out:
-        return ret;
-}
-
-int32_t
-stripe_free_xattr_str (stripe_local_t *local)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!local || !local->xattr_list)
-                goto out;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-
-                if (xattr && xattr->xattr_value)
-                        GF_FREE (xattr->xattr_value);
-        }
-
-        ret = 0;
- out:
-        return ret;
-}
-
-int32_t
-stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            char **xattr_serz)
-{
-        int      ret             = -1;
-        int32_t  padding         = 0;
-        int32_t  tlen            = 0;
-        char stripe_size_str[20] = {0,};
-        char    *pathinfo_serz   = NULL;
-
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
-                goto out;
-        }
-
-        (void) snprintf (stripe_size_str, 20, "%ld",
-                         (local->fctx) ? local->fctx->stripe_size : 0);
-
-        /* extra bytes for decorations (brackets and <>'s) */
-        padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
-                + strlen (stripe_size_str) + 7;
-        local->xattr_total_len += (padding + 2);
-
-        pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
-                                   gf_common_mt_char);
-        if (!pathinfo_serz)
-                goto out;
-
-        /* xlator info */
-        (void) sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ",
-                        this->name, stripe_size_str);
-
-        ret = stripe_xattr_aggregate (pathinfo_serz + padding, local, &tlen);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot aggregate pathinfo list");
-                goto out;
-        }
-
-        *(pathinfo_serz + padding + tlen) = ')';
-        *(pathinfo_serz + padding + tlen + 1) = '\0';
-
-        *xattr_serz = pathinfo_serz;
-
-        ret = 0;
- out:
-        return ret;
-}
-
-int32_t
 stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *dict) {
+                      xlator_t *this, int32_t op_ret, int32_t op_errno,
+                      dict_t *dict, dict_t *xdata)
+{
         stripe_local_t      *local         = NULL;
         int32_t              callcnt       = 0;
         int32_t              ret           = -1;
         long                 cky           = 0;
-        char                *xattr_val     = NULL;
-        char                *xattr_serz    = NULL;
+        void                *xattr_val     = NULL;
+        void                *xattr_serz    = NULL;
         stripe_xattr_sort_t *xattr         = NULL;
         dict_t              *stripe_xattr  = NULL;
 
         if (!frame || !frame->local || !this) {
-                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
+                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
                 return ret;
         }
 
         local = frame->local;
         cky = (long) cookie;
 
-        if (!local->xsel) {
+        if (local->xsel[0] == '\0') {
                 gf_log (this->name, GF_LOG_ERROR, "Empty xattr in cbk");
                 return ret;
         }
@@ -4780,18 +5389,20 @@ stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
                                            gf_stripe_mt_xattr_sort_t);
 
                 if (local->xattr_list) {
-                        ret = dict_get_str (dict, local->xsel, &xattr_val);
-                        if (ret)
-                                goto out;
-
                         xattr = local->xattr_list + (int32_t) cky;
 
-                        xattr_val = gf_strdup (xattr_val);
+                        ret = dict_get_ptr_and_len (dict, local->xsel,
+                                                    &xattr_val,
+                                                    &xattr->xattr_len);
+                        if (xattr->xattr_len == 0)
+                                goto out;
+
                         xattr->pos = cky;
-                        xattr->xattr_value = xattr_val;
-                        xattr->xattr_len = strlen (xattr_val);
+                        xattr->xattr_value = gf_memdup (xattr_val,
+                                                        xattr->xattr_len);
 
-                        local->xattr_total_len += xattr->xattr_len + 1;
+                        if (xattr->xattr_value != NULL)
+                                local->xattr_total_len += xattr->xattr_len + 1;
                 }
         }
  out:
@@ -4808,29 +5419,33 @@ stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
                 /* select filler based on ->xsel */
                 if (XATTR_IS_PATHINFO (local->xsel))
                         ret = stripe_fill_pathinfo_xattr (this, local,
+                                                          (char **)&xattr_serz);
+                else if (XATTR_IS_LOCKINFO (local->xsel)) {
+                        ret = stripe_fill_lockinfo_xattr (this, local,
                                                           &xattr_serz);
-                else {
+                } else {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Unknown xattr in xattr request");
                         goto unwind;
                 }
 
                 if (!ret) {
-                        ret = dict_set_dynstr (stripe_xattr, local->xsel,
-                                               xattr_serz);
+                        ret = dict_set_dynptr (stripe_xattr, local->xsel,
+                                               xattr_serz,
+                                               local->xattr_total_len);
                         if (ret)
                                 gf_log (this->name, GF_LOG_ERROR,
-                                        "Can't set %s key in dict", local->xsel);
+                                        "Can't set %s key in dict",
+                                        local->xsel);
                 }
 
         unwind:
                 STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
-                                     stripe_xattr);
+                                     stripe_xattr, NULL);
 
                 ret = stripe_free_xattr_str (local);
 
-                if (local->xattr_list)
-                        GF_FREE (local->xattr_list);
+                GF_FREE (local->xattr_list);
 
                 if (stripe_xattr)
                         dict_unref (stripe_xattr);
@@ -4841,7 +5456,7 @@ stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 stripe_getxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name)
+                 loc_t *loc, const char *name, dict_t *xdata)
 {
         stripe_local_t    *local    = NULL;
         xlator_list_t     *trav     = NULL;
@@ -4872,7 +5487,7 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
 
 
         if (name && (strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-            && (-1 == frame->root->pid)) {
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
                 local->marker.call_count = priv->child_count;
 
                 sub_volumes = alloca ( priv->child_count *
@@ -4887,7 +5502,8 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
                 if (cluster_getmarkerattr (frame, this, loc, name,
                                            local, stripe_getxattr_unwind,
                                            sub_volumes, priv->child_count,
-                                           MARKER_UUID_TYPE, priv->vol_uuid)) {
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
                         op_errno = EINVAL;
                         goto err;
                 }
@@ -4903,15 +5519,13 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
                              trav = trav->next) {
                         STACK_WIND (frame, stripe_getxattr_cbk,
                                     trav->xlator, trav->xlator->fops->getxattr,
-                                    loc, name);
+                                    loc, name, xdata);
                 }
 
                 return 0;
         }
 
-        if (name &&
-            ((strncmp (name, GF_XATTR_PATHINFO_KEY,
-                       strlen (GF_XATTR_PATHINFO_KEY)) == 0))) {
+        if (name && (XATTR_IS_PATHINFO (name))) {
                 if (IA_ISREG (loc->inode->ia_type)) {
                         ret = inode_ctx_get (loc->inode, this,
                                              (uint64_t *) &local->fctx);
@@ -4935,7 +5549,7 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
                         STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
                                            (void *) (long) i, trav->xlator,
                                            trav->xlator->fops->getxattr,
-                                           loc, name);
+                                           loc, name, xdata);
                 }
 
                 return 0;
@@ -4943,42 +5557,124 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
 
         if (name &&(*priv->vol_uuid)) {
                 if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                    && (-1 == frame->root->pid)) {
-                        local->marker.call_count = priv->child_count;
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+
+                        if (!IA_FILE_OR_DIR (loc->inode->ia_type))
+                                local->marker.call_count = 1;
+                        else
+                                local->marker.call_count = priv->child_count;
 
-                        sub_volumes = alloca ( priv->child_count *
-                                               sizeof (xlator_t *));
-                        for (i = 0, trav = this->children; trav ;
-                             trav = trav->next, i++) {
+                        sub_volumes = alloca (local->marker.call_count *
+                                              sizeof (xlator_t *));
 
+                        for (i = 0, trav = this->children;
+                             i < local->marker.call_count;
+                             i++, trav = trav->next) {
                                 *(sub_volumes + i) = trav->xlator;
 
                         }
 
                         if (cluster_getmarkerattr (frame, this, loc, name,
-                                                   local, stripe_getxattr_unwind,
+                                                   local,
+                                                   stripe_getxattr_unwind,
                                                    sub_volumes,
-                                                   priv->child_count,
+                                                   local->marker.call_count,
                                                    MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
                                                    priv->vol_uuid)) {
                                 op_errno = EINVAL;
                                 goto err;
                         }
+
                         return 0;
                 }
         }
 
 
         STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+
+        return 0;
+
+err:
+        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+inline gf_boolean_t
+stripe_is_special_xattr (const char *name)
+{
+        gf_boolean_t    is_spl = _gf_false;
+
+        if (!name) {
+                goto out;
+        }
+
+        if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                      strlen (GF_XATTR_LOCKINFO_KEY))
+            || XATTR_IS_PATHINFO (name))
+                is_spl = _gf_true;
+out:
+        return is_spl;
+}
+
+int32_t
+stripe_fgetxattr_from_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                const char *name, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv  = NULL;
+        int32_t           ret   = -1, op_errno = 0;
+        int               i     = 0;
+        xlator_list_t    *trav  = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->op_ret = -1;
+        frame->local = local;
+
+        strncpy (local->xsel, name, strlen (name));
+        local->nallocs = local->wind_count = priv->child_count;
+
+        for (i = 0, trav = this->children; i < priv->child_count; i++,
+                     trav = trav->next) {
+                STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
+                                   (void *) (long) i, trav->xlator,
+                                   trav->xlator->fops->fgetxattr,
+                                   fd, name, xdata);
+        }
 
         return 0;
 
 err:
-        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, NULL);
+        return ret;
+}
+
+int32_t
+stripe_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+        if (stripe_is_special_xattr (name)) {
+                stripe_fgetxattr_from_everyone (frame, this, fd, name, xdata);
+                goto out;
+        }
+
+        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+
+out:
         return 0;
 }
 
+
+
 int32_t
 stripe_priv_dump (xlator_t *this)
 {
@@ -5053,9 +5749,13 @@ struct xlator_fops fops = {
         .setxattr       = stripe_setxattr,
         .fsetxattr      = stripe_fsetxattr,
         .getxattr       = stripe_getxattr,
+        .fgetxattr      = stripe_fgetxattr,
         .removexattr    = stripe_removexattr,
         .fremovexattr   = stripe_fremovexattr,
         .readdirp       = stripe_readdirp,
+	.fallocate	= stripe_fallocate,
+	.discard	= stripe_discard,
+        .zerofill       = stripe_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -5069,8 +5769,9 @@ struct xlator_dumpops dumpops = {
 
 struct volume_options options[] = {
         { .key  = {"block-size"},
-          .type = GF_OPTION_TYPE_ANY,
+          .type = GF_OPTION_TYPE_SIZE_LIST,
           .default_value = "128KB",
+          .min = STRIPE_MIN_BLOCK_SIZE,
           .description = "Size of the stripe unit that would be read "
                          "from or written to the striped servers."
         },
@@ -5078,5 +5779,12 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "true"
         },
+	{ .key = {"coalesce"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "true",
+	  .description = "Enable/Disable coalesce mode to flatten striped "
+			 "files as stored on the server (i.e., eliminate holes "
+			 "caused by the traditional format)."
+	},
         { .key  = {NULL} },
 };
diff --git a/xlators/cluster/stripe/src/stripe.h b/xlators/cluster/stripe/src/stripe.h
index 580920a1b..5673d18f3 100644
--- a/xlators/cluster/stripe/src/stripe.h
+++ b/xlators/cluster/stripe/src/stripe.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -38,7 +29,7 @@
 #include <signal.h>
 
 #define STRIPE_PATHINFO_HEADER "STRIPE:"
-
+#define STRIPE_MIN_BLOCK_SIZE  (16*GF_UNIT_KB)
 
 #define STRIPE_STACK_UNWIND(fop, frame, params ...) do {           \
                 stripe_local_t *__local = NULL;                    \
@@ -64,10 +55,27 @@
                 }                                               \
         } while (0)
 
+#define STRIPE_VALIDATE_FCTX(fctx, label) do {                  \
+        int     idx = 0;                                        \
+        if (!fctx) {                                            \
+                op_errno = EINVAL;                              \
+                goto label;                                     \
+        }                                                       \
+        for (idx = 0; idx < fctx->stripe_count; idx++) {        \
+                if (!fctx->xl_array[idx]) {                     \
+                        gf_log (this->name, GF_LOG_ERROR,       \
+                                "fctx->xl_array[%d] is NULL",   \
+                                idx);                           \
+                        op_errno = ESTALE;                      \
+                        goto label;                             \
+                }                                               \
+        }                                                       \
+       } while (0)
+
 typedef struct stripe_xattr_sort {
-        int32_t  pos;
-        int32_t  xattr_len;
-        char    *xattr_value;
+        int   pos;
+        int   xattr_len;
+        char *xattr_value;
 } stripe_xattr_sort_t;
 
 /**
@@ -90,16 +98,17 @@ struct stripe_private {
         gf_lock_t               lock;
         uint8_t                 nodes_down;
         int8_t                  first_child_down;
+        int                    *last_event;
         int8_t                  child_count;
-        int8_t                 *state; /* Current state of child node */
         gf_boolean_t            xattr_supported;  /* default yes */
+	gf_boolean_t		coalesce;
         char                    vol_uuid[UUID_SIZE + 1];
 };
 
 /**
- * Used to keep info about the replies received from fops->readv calls
+ * Used to keep info about the replies received from readv/writev calls
  */
-struct readv_replies {
+struct stripe_replies {
         struct iovec *vector;
         int32_t       count;    //count of vector
         int32_t       op_ret;   //op_ret of readv
@@ -111,6 +120,7 @@ struct readv_replies {
 typedef struct _stripe_fd_ctx {
         off_t      stripe_size;
         int        stripe_count;
+	int	   stripe_coalesce;
         int        static_array;
         xlator_t **xl_array;
 } stripe_fd_ctx_t;
@@ -146,7 +156,7 @@ struct stripe_local {
         blkcnt_t             preparent_blocks;
         blkcnt_t             postparent_blocks;
 
-        struct readv_replies *replies;
+        struct stripe_replies *replies;
         struct statvfs        statvfs_buf;
         dir_entry_t          *entry;
 
@@ -173,7 +183,7 @@ struct stripe_local {
         mode_t               mode;
         dev_t                rdev;
         /* For File I/O fops */
-        dict_t              *dict;
+        dict_t              *xdata;
 
         stripe_xattr_sort_t *xattr_list;
         int32_t              xattr_total_len;
@@ -198,10 +208,81 @@ struct stripe_local {
         gf_dirent_t         *dirent;
         dict_t              *xattr;
         uuid_t               ia_gfid;
+
+        int                  xflag;
+        mode_t               umask;
 };
 
 typedef struct stripe_local   stripe_local_t;
 typedef struct stripe_private stripe_private_t;
 
+/*
+ * Determine the stripe index of a particular frame based on the translator.
+ */
+static inline int32_t stripe_get_frame_index(stripe_fd_ctx_t *fctx,
+					     call_frame_t *prev)
+{
+	int32_t i, idx = -1;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (fctx->xl_array[i] == prev->this) {
+			idx = i;
+			break;
+		}
+	}
+
+	return idx;
+}
+
+static inline void stripe_copy_xl_array(xlator_t **dst, xlator_t **src,
+					int count)
+{
+	int i;
+
+	for (i = 0; i < count; i++)
+		dst[i] = src[i];
+}
+
+void stripe_local_wipe (stripe_local_t *local);
+int32_t stripe_ctx_handle (xlator_t *this, call_frame_t *prev,
+                           stripe_local_t *local, dict_t *dict);
+void stripe_aggregate_xattr (dict_t *dst, dict_t *src);
+int32_t stripe_xattr_request_build (xlator_t *this, dict_t *dict,
+                                    uint64_t stripe_size, uint32_t stripe_count,
+                                    uint32_t stripe_index,
+				    uint32_t stripe_coalesce);
+int32_t stripe_get_matching_bs (const char *path, stripe_private_t *priv);
+int set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data);
+int32_t stripe_iatt_merge (struct iatt *from, struct iatt *to);
+int32_t stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
+                                    char **xattr_serz);
+int32_t stripe_free_xattr_str (stripe_local_t *local);
+int32_t stripe_xattr_aggregate (char *buffer, stripe_local_t *local,
+                                int32_t *total);
+off_t coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count);
+off_t uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
+			int stripe_index);
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz);
+
+/*
+ * Adjust the size attribute for files if coalesce is enabled.
+ */
+static inline void correct_file_size(struct iatt *buf, stripe_fd_ctx_t *fctx,
+	call_frame_t *prev)
+{
+	int index;
+
+	if (!IA_ISREG(buf->ia_type))
+		return;
+
+	if (!fctx || !fctx->stripe_coalesce)
+		return;
+
+	index = stripe_get_frame_index(fctx, prev);
+	buf->ia_size = uncoalesced_size(buf->ia_size, fctx->stripe_size,
+		fctx->stripe_count, index);
+}
 
 #endif /* _STRIPE_H_ */
diff --git a/xlators/cluster/unify/src/Makefile.am b/xlators/cluster/unify/src/Makefile.am
deleted file mode 100644
index 2a1fe8372..000000000
--- a/xlators/cluster/unify/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-
-xlator_LTLIBRARIES = unify.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/legacy/cluster
-
-unify_la_LDFLAGS = -module -avoidversion
-
-unify_la_SOURCES = unify.c unify-self-heal.c
-unify_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = unify.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/unify/src/unify-mem-types.h b/xlators/cluster/unify/src/unify-mem-types.h
deleted file mode 100644
index 13c9cc1f7..000000000
--- a/xlators/cluster/unify/src/unify-mem-types.h
+++ /dev/null
@@ -1,41 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __UNIFY_MEM_TYPES_H__
-#define __UNIFY_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_unify_mem_types_ {
-        gf_unify_mt_char = gf_common_mt_end + 1,
-        gf_unify_mt_int16_t,
-        gf_unify_mt_xlator_t,
-        gf_unify_mt_unify_private_t,
-        gf_unify_mt_xlator_list_t,
-        gf_unify_mt_dir_entry_t,
-        gf_unify_mt_off_t,
-        gf_unify_mt_int,
-        gf_unify_mt_unify_self_heal_struct,
-        gf_unify_mt_unify_local_t,
-        gf_unify_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/unify/src/unify-self-heal.c b/xlators/cluster/unify/src/unify-self-heal.c
deleted file mode 100644
index f99e4c7c3..000000000
--- a/xlators/cluster/unify/src/unify-self-heal.c
+++ /dev/null
@@ -1,1239 +0,0 @@
-/*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * unify-self-heal.c : 
- *   This file implements few functions which enables 'unify' translator 
- *  to be consistent in its behaviour when 
- *     > a node fails, 
- *     > a node gets added, 
- *     > a failed node comes back
- *     > a new namespace server is added (ie, an fresh namespace server).
- * 
- *  This functionality of 'unify' will enable glusterfs to support storage
- *  system failure, and maintain consistancy. This works both ways, ie, when
- *  an entry (either file or directory) is found on namespace server, and not
- *  on storage nodes, its created in storage nodes and vica-versa.
- * 
- *  The two fops, where it can be implemented are 'getdents ()' and 'lookup ()'
- *
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "common-utils.h"
-
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count);
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count);
-
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count);
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count);
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		GF_FREE (local->name);
-	}
-
-	if (local->sh_struct) {
-		if (local->sh_struct->offset_list)
-			GF_FREE (local->sh_struct->offset_list);
-
-		if (local->sh_struct->entry_list)
-			GF_FREE (local->sh_struct->entry_list);
-
-		if (local->sh_struct->count_list)
-			GF_FREE (local->sh_struct->count_list);
-
-		GF_FREE (local->sh_struct);
-	}
-
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-int32_t 
-unify_sh_setdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents on 
-		 * storagenodes is still pending.
-		 */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			if (!entry)
-				return 0;
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more entries
-				   to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_sh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			inode = local->loc1.inode;
-			fd_unref (local->fd);
-			tmp_dict = local->dict;
-
-			unify_local_wipe (local);
-			
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      inode, &local->stbuf, local->dict,
-                                      &local->oldpostparent);
-			if (tmp_dict)
-				dict_unref (tmp_dict);
-		}
-	}
-  
-	return 0;
-}
-
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                      gf_unify_mt_dir_entry_t);
-	
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, it 
-		   can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_sh_ns_setdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		if (local->sh_struct->entry_list[index]) {
-			prev = entry = local->sh_struct->entry_list[index];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_sh_getdents_cbk -
- */
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                 gf_unify_mt_dir_entry_t);
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_sh_opendir_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_WARNING, "failed");
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing
-			   storage nodes */
-			local->call_count--; 
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps
-				 * track of offset sent to each node during
-				 * STACK_WIND.
-				 */
-				local->sh_struct->offset_list = 
-					GF_CALLOC (priv->child_count, 
-					           sizeof (off_t),
-                                                   gf_unify_mt_off_t);
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (dir_entry_t *),
-                                                   gf_unify_mt_dir_entry_t);
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (int),
-                                                   gf_unify_mt_int);
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_sh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-
-				/* did stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. */ 
-		inode = local->loc1.inode;
-		fd_unref (local->fd);
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-		/* Only 'self-heal' failed, lookup() was successful. */
-		local->op_ret = 0;
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, inode,
-			      &local->stbuf, local->dict, &local->oldpostparent);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * gf_sh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_sh_checksum_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       uint8_t *file_checksum,
-		       uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, NAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, NAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-                                        /* Using 'entry_count' as a flag */
-					local->entry_count = 1;
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, NAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < NAME_MAX; index++) {
-					/* Files should be present in
-					   only one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be
-					   same accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < NAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at directory level */
-			local->call_count = 0;
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		inode = local->loc1.inode;
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      inode,
-			      &local->stbuf,
-			      local->dict, &local->oldpostparent);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/* Foreground self-heal part over */
-
-/* Background self-heal part */
-
-int32_t 
-unify_bgsh_setdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents 
-		   on storagenodes is still pending. */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more
-				   entries to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_bgsh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			fd_unref (local->fd);
-			unify_local_wipe (local);
-			STACK_DESTROY (frame->root);
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                      gf_unify_mt_dir_entry_t);
-
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, 
-		   it can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_bgsh_ns_setdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	if (local->sh_struct->entry_list[index]) {
-		prev = entry = local->sh_struct->entry_list[index];
-		if (!entry)
-			return 0;
-		trav = entry->next;
-		while (trav) {
-			prev->next = trav->next;
-			GF_FREE (trav->name);
-			if (IA_ISLNK (trav->buf.ia_type))
-				GF_FREE (trav->link);
-			GF_FREE (trav);
-			trav = prev->next;
-		}
-		GF_FREE (entry);
-	}
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_bgsh_getdents_cbk -
- */
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                 gf_unify_mt_dir_entry_t);
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for 
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_bgsh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_bgsh_opendir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			fd_t *fd)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int32_t callcnt = 0;
-	int16_t index = 0;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing 
-			   storage nodes */
-			local->call_count--; 
-			callcnt = local->call_count;
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps 
-				   track of offset sent to each node during 
-				   STACK_WIND. */
-				local->sh_struct->offset_list = 
-					GF_CALLOC (priv->child_count, 
-					           sizeof (off_t),
-                                                   gf_unify_mt_off_t);
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (dir_entry_t *),
-                                                   gf_unify_mt_dir_entry_t);
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (int),
-                                                   gf_unify_mt_int);
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_bgsh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-				/* did a stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. 	 */
-		fd_unref (local->fd);
-		
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/**
- * gf_bgsh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_bgsh_checksum_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 uint8_t *file_checksum,
-			 uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, NAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, NAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other 
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-					/* Using 'entry_count' as a flag */
-					local->entry_count = 1; 
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, NAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < NAME_MAX; index++) {
-					/* Files should be present in only 
-					   one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be same 
-					   accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < NAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at the directory level */
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_bgsh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-      
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/* Background self-heal part over */
-
-
-
-
-/**
- * zr_unify_self_heal - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-zr_unify_self_heal (call_frame_t *frame,
-		    xlator_t *this,
-		    unify_local_t *local)
-{
-	unify_private_t *priv = this->private;
-	call_frame_t *bg_frame = NULL;
-	unify_local_t *bg_local = NULL;
-	inode_t *tmp_inode = NULL;
-	dict_t *tmp_dict = NULL;
-	int16_t index = 0;
-  
-	if (local->inode_generation < priv->inode_generation) {
-		/* Any self heal will be done at the directory level */
-		/* Update the inode's generation to the current generation
-		   value. */
-		local->inode_generation = priv->inode_generation;
-		inode_ctx_put (local->loc1.inode, this, 
-			  (uint64_t)(long)local->inode_generation);
-
-		if (priv->self_heal == ZR_UNIFY_FG_SELF_HEAL) {
-			local->op_ret = 0;
-			local->failed = 0;
-			local->call_count = priv->child_count + 1;
-			local->sh_struct = 
-	         	     GF_CALLOC (1, sizeof (struct unify_self_heal_struct),
-                                        gf_unify_mt_unify_self_heal_struct);
-      
-			/* +1 is for NS */
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_checksum_cbk,
-						   priv->xl_array[index],
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->checksum,
-						   &local->loc1,
-						   0);
-			}
-
-			/* Self-heal in foreground, hence no need 
-			   to UNWIND here */
-			return 0;
-		}
-
-		/* Self Heal done in background */
-		bg_frame = copy_frame (frame);
-		INIT_LOCAL (bg_frame, bg_local);
-		loc_copy (&bg_local->loc1, &local->loc1);
-		bg_local->op_ret = 0;
-		bg_local->failed = 0;
-		bg_local->call_count = priv->child_count + 1;
-		bg_local->sh_struct = 
-			GF_CALLOC (1, sizeof (struct unify_self_heal_struct),
-                                   gf_unify_mt_unify_self_heal_struct);
-    
-		/* +1 is for NS */
-		for (index = 0; index < (priv->child_count + 1); index++) {
-			STACK_WIND_COOKIE (bg_frame,
-					   unify_bgsh_checksum_cbk,
-					   priv->xl_array[index],
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->checksum,
-					   &bg_local->loc1,
-					   0);
-		}
-	}
-
-	/* generation number matches, self heal already done or
-	 * self heal done in background: just do STACK_UNWIND 
-	 */
-	tmp_inode = local->loc1.inode;
-	tmp_dict = local->dict;
-
-	unify_local_wipe (local);
-
-	/* This is lookup_cbk ()'s UNWIND. */
-	STACK_UNWIND (frame,
-		      local->op_ret,
-		      local->op_errno,
-		      tmp_inode,
-		      &local->stbuf,
-		      local->dict,
-                      &local->oldpostparent);
-
-	if (tmp_dict)
-		dict_unref (tmp_dict);
-
-	return 0;
-}
-
diff --git a/xlators/cluster/unify/src/unify.c b/xlators/cluster/unify/src/unify.c
deleted file mode 100644
index 6dc93083d..000000000
--- a/xlators/cluster/unify/src/unify.c
+++ /dev/null
@@ -1,4589 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * xlators/cluster/unify:
- *     - This xlator is one of the main translator in GlusterFS, which
- *   actually does the clustering work of the file system. One need to 
- *   understand that, unify assumes file to be existing in only one of 
- *   the child node, and directories to be present on all the nodes. 
- *
- * NOTE:
- *   Now, unify has support for global namespace, which is used to keep a 
- * global view of fs's namespace tree. The stat for directories are taken
- * just from the namespace, where as for files, just 'ia_ino' is taken from
- * Namespace node, and other stat info is taken from the actual storage node.
- * Also Namespace node helps to keep consistant inode for files across 
- * glusterfs (re-)mounts.
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include <signal.h>
-#include <libgen.h>
-#include "compat-errno.h"
-#include "compat.h"
-
-#define UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR(_loc) do { \
-  if (!(_loc && _loc->inode)) {                            \
-    STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL);    \
-    return 0;                                              \
-  }                                                        \
-} while(0)
-
-
-#define UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!(_fd && !fd_ctx_get (_fd, this, NULL))) {       \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);      \
-    return 0;                                          \
-  }                                                    \
-} while(0)
-
-#define UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!_fd) {                                      \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);  \
-    return 0;                                      \
-  }                                                \
-} while(0)
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		GF_FREE (local->name);
-	}
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-
-
-/*
- * unify_normalize_stats -
- */
-void
-unify_normalize_stats (struct statvfs *buf,
-		       unsigned long bsize,
-		       unsigned long frsize)
-{
-	double factor;
-
-	if (buf->f_bsize != bsize) {
-		factor = ((double) buf->f_bsize) / bsize;
-		buf->f_bsize  = bsize;
-		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-	}
-  
-	if (buf->f_frsize != frsize) {
-		factor = ((double) buf->f_frsize) / frsize;
-		buf->f_frsize = frsize;
-		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-	}
-}
-
-
-xlator_t *
-unify_loc_subvol (loc_t *loc, xlator_t *this)
-{
-	unify_private_t *priv = NULL;
-	xlator_t        *subvol = NULL;
-	int16_t         *list = NULL;
-	long             index = 0;
-	xlator_t        *subvol_i = NULL;
-	int              ret = 0;
-	uint64_t         tmp_list = 0;
-
-	priv   = this->private;
-	subvol = NS (this);
-
-	if (!IA_ISDIR (loc->inode->ia_type)) {
-		ret = inode_ctx_get (loc->inode, this, &tmp_list);
-		list = (int16_t *)(long)tmp_list;
-		if (!list)
-			goto out;
-
-		for (index = 0; list[index] != -1; index++) {
-			subvol_i = priv->xl_array[list[index]];
-			if (subvol_i != NS (this)) {
-				subvol = subvol_i;
-				break;
-			}
-		}
-	}
-out:
-	return subvol;
-}
-
-
-
-/**
- * unify_statfs_cbk -
- */
-int32_t
-unify_statfs_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct statvfs *stbuf)
-{
-	int32_t callcnt = 0;
-	struct statvfs *dict_buf = NULL;
-	unsigned long bsize;
-	unsigned long frsize;
-	unify_local_t *local = (unify_local_t *)frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			/* when a call is successfull, add it to local->dict */
-			dict_buf = &local->statvfs_buf;
-
-			if (dict_buf->f_bsize != 0) {
-				bsize  = max (dict_buf->f_bsize, 
-					      stbuf->f_bsize);
-
-				frsize = max (dict_buf->f_frsize, 
-					      stbuf->f_frsize);
-				unify_normalize_stats(dict_buf, bsize, frsize);
-				unify_normalize_stats(stbuf, bsize, frsize);
-			} else {
-				dict_buf->f_bsize   = stbuf->f_bsize;
-				dict_buf->f_frsize  = stbuf->f_frsize;
-			}
-      
-			dict_buf->f_blocks += stbuf->f_blocks;
-			dict_buf->f_bfree  += stbuf->f_bfree;
-			dict_buf->f_bavail += stbuf->f_bavail;
-			dict_buf->f_files  += stbuf->f_files;
-			dict_buf->f_ffree  += stbuf->f_ffree;
-			dict_buf->f_favail += stbuf->f_favail;
-			dict_buf->f_fsid    = stbuf->f_fsid;
-			dict_buf->f_flag    = stbuf->f_flag;
-			dict_buf->f_namemax = stbuf->f_namemax;
-			local->op_ret = op_ret;
-		} else {
-			/* fop on storage node has failed due to some error */
-			if (op_errno != ENOTCONN) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): %s", 
-					prev_frame->this->name, 
-					strerror (op_errno));
-			}
-			local->op_errno = op_errno;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->statvfs_buf);
-	}
-
-	return 0;
-}
-
-/**
- * unify_statfs -
- */
-int32_t
-unify_statfs (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-
-	INIT_LOCAL (frame, local);
-	local->call_count = ((unify_private_t *)this->private)->child_count;
-
-	while(trav) {
-		STACK_WIND (frame,
-			    unify_statfs_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->statfs,
-			    loc);
-		trav = trav->next;
-	}
-
-	return 0;
-}
-
-/**
- * unify_buf_cbk - 
- */
-int32_t
-unify_buf_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct iatt *buf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = buf->ia_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (IA_ISDIR (buf->ia_type) || 
-				    !local->stbuf.ia_blksize) {
-					local->stbuf = *buf;
-				}
-			}
-
-			if ((!IA_ISDIR (buf->ia_type)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->ia_ino)
-			local->op_ret = -1;
-
-		local->stbuf.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-
-	return 0;
-}
-
-#define check_if_dht_linkfile(s) \
-        ((st_mode_from_ia (s->ia_prot, s->ia_type) & ~S_IFMT) == S_ISVTX)
-
-/**
- * unify_lookup_cbk - 
- */
-int32_t 
-unify_lookup_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-		  struct iatt *buf,
-		  dict_t *dict,
-                  struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-	dict_t *local_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
- 
-		if (op_ret == -1) {
-                        if (local->revalidate &&
-                            (op_errno == ESTALE)) {
-                                /* ESTALE takes priority */
-                                local->op_errno = op_errno;
-                                local->failed = 1;
-                        }
-
-			if ((op_errno != ENOTCONN) 
-                            && (op_errno != ENOENT)
-                            && (local->op_errno != ESTALE)) {
-                                /* if local->op_errno is already ESTALE, then
-                                 * ESTALE has to propogated to the parent first.
-                                 * do not enter here.
-                                 */
-				gf_log (this->name, GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-                                local->op_errno = op_errno;
-				local->failed = 1;
-
-			} else if (local->revalidate && 
-                                   (local->op_errno != ESTALE) &&
-				   !(priv->optimist && (op_errno == ENOENT))) {
-
-				gf_log (this->name, 
-					(op_errno == ENOTCONN) ? 
-					GF_LOG_DEBUG:GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-				local->op_errno = op_errno;
-				local->failed = 1;
-			}
-		}
-
-		if (op_ret == 0) {
-			local->op_ret = 0; 
-			
-			if (check_if_dht_linkfile(buf)) {
-				gf_log (this->name, GF_LOG_CRITICAL,
-					"file %s may be DHT link file on %s, "
-					"make sure the backend is not shared "
-					"between unify and DHT", 
-					local->loc1.path, 
-					priv->xl_array[(long)cookie]->name);
-			}
-
-			if (local->stbuf.ia_type && local->stbuf.ia_blksize) {
-				/* make sure we already have a stbuf
-				   stored in local->stbuf */
-				if (IA_ISDIR (local->stbuf.ia_type) && 
-				    !IA_ISDIR (buf->ia_type)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on namespace, non-directory "
-						"on node '%s', returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-				if (!IA_ISDIR (local->stbuf.ia_type) && 
-				    IA_ISDIR (buf->ia_type)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on node '%s', non-directory "
-						"on namespace, returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-			}
-	
-			if (!local->revalidate && !IA_ISDIR (buf->ia_type)) {
-				/* This is the first time lookup on file*/
-				if (!local->list) {
-					/* list is not allocated, allocate 
-					   the max possible range */
-					local->list = GF_CALLOC (1, 2 * (priv->child_count + 2),
-                                                                 gf_unify_mt_int16_t);
-					if (!local->list) {
-						gf_log (this->name, 
-							GF_LOG_CRITICAL, 
-							"Not enough memory");
-						STACK_UNWIND (frame, -1, 
-							      ENOMEM, inode, 
-							      NULL, NULL, NULL);
-						return 0;
-					}
-				}
-				/* update the index of the list */
-				local->list [local->index++] = 
-					(int16_t)(long)cookie;
-			}
-                        
-                        if (!local->revalidate && IA_ISDIR (buf->ia_type)) {
-                                /* fresh lookup of a directory */
-                                inode_ctx_put (local->loc1.inode, this, 
-                                               priv->inode_generation);
-                        }
-
-			if ((!local->dict) && dict &&
-			    (priv->xl_array[(long)cookie] != NS(this)))	{
-				local->dict = dict_ref (dict);
-			}
-
-			/* index of NS node is == total child count */
-			if (priv->child_count == (int16_t)(long)cookie) {
-				/* Take the inode number from namespace */
-				local->ia_ino = buf->ia_ino;
-				if (IA_ISDIR (buf->ia_type) || 
-				    !(local->stbuf.ia_blksize)) {
-					local->stbuf = *buf;
-                                        local->oldpostparent = *postparent;
-				}
-			} else if (!IA_ISDIR (buf->ia_type)) {
-				/* If file, then get the stat from 
-				   storage node */
-				local->stbuf = *buf;
-			}
-
-			if (local->ia_nlink < buf->ia_nlink) {
-				local->ia_nlink = buf->ia_nlink;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local_dict = local->dict;
-		if (local->return_eio) {
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"[CRITICAL] Unable to fix the path (%s) with "
-				"self-heal, try manual verification. "
-				"returning EIO.", local->loc1.path);
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EIO, inode, NULL, NULL);
-			if (local_dict)	{
-				dict_unref (local_dict);
-			}
-			return 0;
-		}
-
-		if (!local->stbuf.ia_blksize) {
-			/* Inode not present */
-			local->op_ret = -1;
-		} else {
-			if (!local->revalidate && 
-			    !IA_ISDIR (local->stbuf.ia_type)) { 
-				/* If its a file, big array is useless, 
-				   allocate the smaller one */
-				int16_t *list = NULL;
-				list = GF_CALLOC (1, 2 * (local->index + 1),
-                                                  gf_unify_mt_int16_t);
-				ERR_ABORT (list);
-				memcpy (list, local->list, 2 * local->index);
-				/* Make the end of the list as -1 */
-				GF_FREE (local->list);
-				local->list = list;
-				local->list [local->index] = -1;
-				/* Update the inode's ctx with proper array */
-				/* TODO: log on failure */
-				inode_ctx_put (local->loc1.inode, this, 
-					       (uint64_t)(long)local->list);
-			}
-
-			if (IA_ISDIR(local->loc1.inode->ia_type)) {
-				/* lookup is done for directory */
-				if (local->failed && priv->self_heal) {
-					/* Triggering self-heal */
-                                        /* means, self-heal required for this 
-					   inode */
-					local->inode_generation = 0; 
-					priv->inode_generation++;
-				}
-			} else {
-				local->stbuf.ia_ino = local->ia_ino;
-			}
-	  
-			local->stbuf.ia_nlink = local->ia_nlink;
-		}
-		if (local->op_ret == -1) {
-			if (!local->revalidate && local->list)
-				GF_FREE (local->list);
-		}
-
-		if ((local->op_ret >= 0) && local->failed && 
-		    local->revalidate) {
-			/* Done revalidate, but it failed */
-			if ((op_errno != ENOTCONN) 
-                            && (local->op_errno != ESTALE)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Revalidate failed for path(%s): %s", 
-					local->loc1.path, strerror (op_errno));
-			}
-			local->op_ret = -1;
-		}
-
-		if ((priv->self_heal && !priv->optimist) && 
-		    (!local->revalidate && (local->op_ret == 0) && 
-		     IA_ISDIR(local->stbuf.ia_type))) {
-			/* Let the self heal be done here */
-			zr_unify_self_heal (frame, this, local);
-			local_dict = NULL;
-		} else {
-                        if (local->failed) {
-                                /* NOTE: directory lookup is sent to all 
-                                 * subvolumes and success from a subvolume
-                                 * might set local->op_ret to 0 (zero) */
-                                local->op_ret = -1;
-                        }
-
-			/* either no self heal, or op_ret == -1 (failure) */
-			tmp_inode = local->loc1.inode;
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      tmp_inode, &local->stbuf, local->dict,
-                                      &local->oldpostparent);
-		}
-		if (local_dict) {
-			dict_unref (local_dict);
-		}
-	}
-  
-	return 0;
-}
-
-/**
- * unify_lookup - 
- */
-int32_t 
-unify_lookup (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      dict_t *xattr_req)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	long index = 0;
-
-	if (!(loc && loc->inode)) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: Argument not right", loc?loc->path:"(null)");
-		STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL, NULL, NULL);
-		return 0;
-	}
-        
-        if (inode_ctx_get (loc->inode, this, NULL)
-            && IA_ISDIR (loc->inode->ia_type)) {
-                local->revalidate = 1;
-        }
-
-	if (!inode_ctx_get (loc->inode, this, NULL) && 
-	    loc->inode->ia_type && 
-	    !IA_ISDIR (loc->inode->ia_type)) {
-		uint64_t tmp_list = 0;
-		/* check if revalidate or fresh lookup */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-	}
-
-	if (local->list) {
-		list = local->list;
-		for (index = 0; list[index] != -1; index++);
-		if (index != 2) {
-			if (index < 2) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning ESTALE for %s: file "
-					"count is %ld", loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-				unify_local_wipe (local);
-				STACK_UNWIND (frame, -1, ESTALE, 
-					      NULL, NULL, NULL, NULL);
-				return 0;  
-			} else {
-				/* There are more than 2 presences */
-				/* Just log and continue */
-				gf_log (this->name, GF_LOG_ERROR,
-					"%s: file count is %ld", 
-					loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-			}
-		}
-      
-		/* is revalidate */
-		local->revalidate = 1;
-      
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-      
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)(long)list[index], //cookie
-					   priv->xl_array [list[index]],
-					   priv->xl_array [list[index]]->fops->lookup,
-					   loc,
-					   xattr_req);
-			if (need_break)
-				break;
-		}
-	} else {
-		if (loc->inode->ia_type) {
-			if (inode_ctx_get (loc->inode, this, NULL)) {
-				inode_ctx_get (loc->inode, this, 
-					       &local->inode_generation);
-			}
-		}
-		/* This is first call, there is no list */
-		/* call count should be all child + 1 namespace */
-		local->call_count = priv->child_count + 1;
-      
-		for (index = 0; index <= priv->child_count; index++) {
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)index, //cookie
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   loc,
-					   xattr_req);
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_stat - if directory, get the stat directly from NameSpace child.
- *     if file, check for a hint and send it only there (also to NS).
- *     if its a fresh stat, then do it on all the nodes.
- *
- * NOTE: for all the call, sending cookie as xlator pointer, which will be 
- *       used in cbk.
- */
-int32_t
-unify_stat (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int16_t *list = NULL;
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL);
-		return 0;
-	}
-	local->ia_ino = loc->inode->ino;
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		/* Directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->stat, loc);
-	} else {
-		/* File */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-    		list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-    
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->stat,
-				    loc);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_access_cbk -
- */
-int32_t
-unify_access_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_access - Send request to only namespace, which has all the 
- *      attributes set for the file.
- */
-int32_t
-unify_access (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t mask)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame,
-		    unify_access_cbk,
-		    NS(this),
-		    NS(this)->fops->access,
-		    loc,
-		    mask);
-
-	return 0;
-}
-
-int32_t
-unify_mkdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-  
-		if ((op_ret == -1) && !(priv->optimist && 
-					(op_errno == ENOENT || 
-					 op_errno == EEXIST))) {
-			/* TODO: Decrement the inode_generation of 
-			 * this->inode's parent inode, hence the missing 
-			 * directory is created properly by self-heal. 
-			 * Currently, there is no way to get the parent 
-			 * inode directly.
-			 */
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			if (op_errno != EEXIST)
-				local->failed = 1;
-			local->op_errno = op_errno;
-		}
-  
-		if (op_ret >= 0)
-			local->op_ret = 0;
-
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (!local->failed) {
-			inode_ctx_put (local->loc1.inode, this, 
-				       priv->inode_generation);
-		}
-		
-		tmp_inode = local->loc1.inode;
-		unify_local_wipe (local);
-
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      tmp_inode, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_mkdir_cbk -
- */
-int32_t
-unify_ns_mkdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	long index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send mkdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR,
-			"namespace: path(%s): %s", 
-			local->name, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	local->call_count = priv->child_count;
-
-	/* Send mkdir request to all the nodes now */
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND_COOKIE (frame,
-				   unify_mkdir_cbk,
-				   (void *)index, //cookie
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->mkdir,
-				   &local->loc1,
-				   local->mode);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_mkdir -
- */
-int32_t
-unify_mkdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode)
-{
-	unify_local_t *local = NULL;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-
-	loc_copy (&local->loc1, loc);
-
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mkdir_cbk,
-		    NS(this),
-		    NS(this)->fops->mkdir,
-		    loc,
-		    mode);
-	return 0;
-}
-
-/**
- * unify_rmdir_cbk -
- */
-int32_t
-unify_rmdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0 || (priv->optimist && (op_errno == ENOENT)))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_rmdir_cbk -
- */
-int32_t
-unify_ns_rmdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-  
-	if (op_ret == -1) {
-		/* No need to send rmdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, 
-			((op_errno != ENOTEMPTY) ? 
-			 GF_LOG_ERROR : GF_LOG_DEBUG),
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, NULL);
-		return 0;
-	}
-
-	local->call_count = priv->child_count;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND (frame,
-			    unify_rmdir_cbk,
-			    priv->xl_array[index],
-			    priv->xl_array[index]->fops->rmdir,
-			    &local->loc1);
-	}
-
-	return 0;
-}
-
-/**
- * unify_rmdir -
- */
-int32_t
-unify_rmdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	unify_local_t *local = NULL;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_rmdir_cbk,
-		    NS(this),
-		    NS(this)->fops->rmdir,
-		    loc);
-
-	return 0;
-}
-
-/**
- * unify_open_cbk -
- */
-int32_t
-unify_open_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in 
-				   all the f*** / FileIO calls */
-				fd_ctx_set (fd, this, (uint64_t)(long)cookie);
-			}
-		}
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if ((local->failed == 1) && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			//local->op_errno = EIO; 
-      
-			if (!fd_ctx_get (local->fd, this, NULL)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on child node, "
-					"failed on namespace");
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on namespace, "
-					"failed on child node");
-			}
-		}
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->fd);
-	}
-
-	return 0;
-}
-
-#ifdef GF_DARWIN_HOST_OS
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_open_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct iatt *buf,
-		       dict_t *dict,
-		       struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if ((op_ret == -1) && (op_errno != ENOENT)) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->index++;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->list[0] = (int16_t)(long)cookie;
-			} else {
-				local->list[1] = (int16_t)(long)cookie;
-			}
-			if (IA_ISDIR (buf->ia_type))
-				local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		file_list[0] = local->list[0];
-		file_list[1] = local->list[1];
-		file_list[2] = -1;
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->name, local->index);
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning as file found on less "
-					"than 2 nodes");
-				STACK_UNWIND (frame, local->op_ret, 
-					      local->op_errno, local->fd);
-				return 0;
-			}
-		}
-
-		if (local->failed) {
-			/* Open on directory, return EISDIR */
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EISDIR, local->fd);
-			return 0;
-		}
-
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd, local->wbflags);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_open_readlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 const char *path,
-			 struct iatt *sbuf)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		STACK_UNWIND (frame, -1, ENOENT);
-		return 0;
-	}
-
-	if (path[0] == '/') {
-		local->name = gf_strdup (path);
-		ERR_ABORT (local->name);
-	} else {
-		char *tmp_str = gf_strdup (local->loc1.path);
-		char *tmp_base = dirname (tmp_str);
-		local->name = GF_CALLOC (1, ZR_PATH_MAX, gf_unify_mt_char);
-		strcpy (local->name, tmp_base);
-		strncat (local->name, "/", 1);
-		strcat (local->name, path);
-		GF_FREE (tmp_str);
-	}
-  
-	local->list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                 gf_unify_mt_int16_t);
-	ERR_ABORT (local->list);
-	local->call_count = priv->child_count + 1;
-	local->op_ret = -1;
-	for (index = 0; index <= priv->child_count; index++) {
-		/* Send the lookup to all the nodes including namespace */
-		STACK_WIND_COOKIE (frame,
-				   unify_open_lookup_cbk,
-				   (void *)(long)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->lookup,
-				   &local->loc1,
-				   NULL);
-	}
-
-	return 0;
-}
-#endif /* GF_DARWIN_HOST_OS */
-
-/**
- * unify_open - 
- */
-int32_t
-unify_open (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t flags,
-	    fd_t *fd,
-            int32_t wbflags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t file_list[3] = {0,};
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Init */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->fd    = fd;
-	local->flags = flags;
-        local->wbflags = wbflags;
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	local->list = list;
-	file_list[0] = priv->child_count; /* Thats namespace */
-	file_list[2] = -1;
-	for (index = 0; list[index] != -1; index++) {
-		local->call_count++;
-		if (list[index] != priv->child_count)
-			file_list[1] = list[index];
-	}
-
-	if (local->call_count != 2) {
-		/* If the lookup was done for file */
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: entry_count is %d",
-			loc->path, local->call_count);
-		for (index = 0; local->list[index] != -1; index++)
-			gf_log (this->name, GF_LOG_ERROR, "%s: found on %s",
-				loc->path, priv->xl_array[list[index]]->name);
-
-		if (local->call_count < 2) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"returning EIO as file found on onlyone node");
-			STACK_UNWIND (frame, -1, EIO, fd);
-			return 0;
-		}
-	}
-
-#ifdef GF_DARWIN_HOST_OS
-	/* Handle symlink here */
-	if (IA_ISLNK (loc->inode->ia_type)) {
-		/* Callcount doesn't matter here */
-		STACK_WIND (frame,
-			    unify_open_readlink_cbk,
-			    NS(this),
-			    NS(this)->fops->readlink,
-			    loc, ZR_PATH_MAX);
-		return 0;
-	}
-#endif /* GF_DARWIN_HOST_OS */
-
-	local->call_count = 2;
-	for (index = 0; file_list[index] != -1; index++) {
-		char need_break = (file_list[index+1] == -1);
-		STACK_WIND_COOKIE (frame,
-				   unify_open_cbk,
-				   priv->xl_array[file_list[index]], //cookie
-				   priv->xl_array[file_list[index]],
-				   priv->xl_array[file_list[index]]->fops->open,
-				   loc,
-				   flags,
-				   fd, wbflags);
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-
-int32_t 
-unify_create_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-	inode_t *inode = local->loc1.inode;
-
-	unify_local_wipe (local);
-
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-  
-	return 0;
-}
-
-/**
- * unify_create_open_cbk -
- */
-int32_t
-unify_create_open_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       fd_t *fd)
-{
-	int ret = 0;
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_value = 0;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in all 
-				   the f*** / FileIO calls */
-				/* TODO: log on failure */
-				ret = fd_ctx_get (fd, this, &tmp_value);
-				cookie = (void *)(long)tmp_value;
-			} else {
-				/* NOTE: open successful on namespace.
-				 *       fd's ctx can be used to identify open 
-				 *       failure on storage subvolume. cool 
-				 *       ide ;) */
-				local->failed = 0;
-			}
-		} else {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				((xlator_t *)cookie)->name,
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed == 1 && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			local->fd = fd;
-			local->call_count = 1;
-
-			if (!fd_ctx_get (local->fd, this, &tmp_value)) {
-				child = (xlator_t *)(long)tmp_value;
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on child node, "
-					"failed on namespace");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    child,
-					    child->fops->unlink,
-					    &local->loc1);
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on namespace, "
-					"failed on child node");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    NS(this),
-					    NS(this)->fops->unlink,
-					    &local->loc1);
-			}
-			return 0;
-		}
-		inode = local->loc1.inode;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, fd,
-			      inode, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-	return 0;
-}
-
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_create_lookup_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 inode_t *inode,
-			 struct iatt *buf,
-			 dict_t *dict,
-                         struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->list[local->index++] = (int16_t)(long)cookie;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->ia_ino = buf->ia_ino;
-			} else {
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t *list = local->list;
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		local->list [local->index] = -1;
-		file_list[0] = list[0];
-		file_list[1] = list[1];
-		file_list[2] = -1;
-
-		local->stbuf.ia_ino = local->ia_ino;
-		/* TODO: log on failure */
-		inode_ctx_put (local->loc1.inode, this, 
-			       (uint64_t)(long)local->list);
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->loc1.path, local->index);
-			file_list[0] = priv->child_count;
-			for (index = 0; list[index] != -1; index++) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"%s: found on %s", local->loc1.path, 
-					priv->xl_array[list[index]]->name);
-				if (list[index] != priv->child_count)
-					file_list[1] = list[index];
-			}
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning EIO as file found on "
-					"only one node");
-				STACK_UNWIND (frame, -1, EIO, 
-					      local->fd, inode, NULL,
-                                              NULL, NULL);
-				return 0;
-			}
-		}
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_create_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd, 0);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_create_cbk -
- */
-int32_t
-unify_create_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd,
-		  inode_t *inode,
-		  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int ret = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	inode_t *tmp_inode = NULL;
-
-	if (op_ret == -1) {
-		/* send unlink () on Namespace */
-		local->op_errno = op_errno;
-		local->op_ret = -1;
-		local->call_count = 1;
-		gf_log (this->name, GF_LOG_ERROR,
-			"create failed on %s (file %s, error %s), "
-			"sending unlink to namespace", 
-			prev_frame->this->name, 
-			local->loc1.path, strerror (op_errno));
-
-		STACK_WIND (frame,
-			    unify_create_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-
-		return 0;
-	}
-
-	if (op_ret >= 0) {
-		local->op_ret = op_ret;
-		local->stbuf = *buf;
-		/* Just inode number should be from NS node */
-		local->stbuf.ia_ino = local->ia_ino;
-
-		/* TODO: log on failure */
-		ret = fd_ctx_set (fd, this, (uint64_t)(long)prev_frame->this);
-	}
-  
-	tmp_inode = local->loc1.inode;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      tmp_inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_create_cbk -
- * 
- */
-int32_t
-unify_ns_create_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     struct iatt *preparent,
-                     struct iatt *postparent)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send create request to other servers, as 
-		   namespace action failed. Handle exclusive create here. */
-		if ((op_errno != EEXIST) || 
-		    ((op_errno == EEXIST) && 
-		     ((local->flags & O_EXCL) == O_EXCL))) {
-			/* If its just a create call without O_EXCL, 
-			   don't do this */
-			gf_log (this->name, GF_LOG_ERROR,
-				"namespace: path(%s): %s", 
-				local->loc1.path, strerror (op_errno));
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf,
-                                      preparent, postparent);
-			return 0;
-		}
-	}
-  
-	if (op_ret >= 0) {
-		/* Get the inode number from the NS node */
-		local->ia_ino = buf->ia_ino;
-  
-                local->oldpreparent = *preparent;
-                local->oldpostparent = *postparent;
-
-		local->op_ret = -1;
-
-		/* Start the mapping list */
-		list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                  gf_unify_mt_int16_t);
-		ERR_ABORT (list);
-		inode_ctx_put (inode, this, (uint64_t)(long)list);
-		list[0] = priv->child_count;
-		list[2] = -1;
-
-		/* This means, file doesn't exist anywhere in the Filesystem */
-		sched_ops = priv->sched_ops;
-
-		/* Send create request to the scheduled node now */
-		sched_xl = sched_ops->schedule (this, local->loc1.path);
-		if (sched_xl == NULL)
-		{
-			/* send unlink () on Namespace */
-			local->op_errno = ENOTCONN;
-			local->op_ret = -1;
-			local->call_count = 1;
-			gf_log (this->name, GF_LOG_ERROR,
-				"no node online to schedule create:(file %s) "
-				"sending unlink to namespace", 
-				(local->loc1.path)?local->loc1.path:"");
-
-			STACK_WIND (frame,
-				    unify_create_unlink_cbk,
-				    NS(this),
-				    NS(this)->fops->unlink,
-				    &local->loc1);
-	
-			return 0;
-		}
-
-		for (index = 0; index < priv->child_count; index++)
-			if (sched_xl == priv->xl_array[index])
-				break;
-		list[1] = index;
-
-		STACK_WIND (frame, unify_create_cbk,
-			    sched_xl, sched_xl->fops->create,
-			    &local->loc1, local->flags, local->mode, fd);
-	} else {
-		/* File already exists, and there is no O_EXCL flag */
-
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"File(%s) already exists on namespace, sending "
-			"open instead", local->loc1.path);
-
-		local->list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                         gf_unify_mt_int16_t);
-		ERR_ABORT (local->list);
-		local->call_count = priv->child_count + 1;
-		local->op_ret = -1;
-		for (index = 0; index <= priv->child_count; index++) {
-			/* Send lookup() to all nodes including namespace */
-			STACK_WIND_COOKIE (frame,
-					   unify_create_lookup_cbk,
-					   (void *)(long)index,
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   &local->loc1,
-					   NULL);
-		}
-	}
-	return 0;
-}
-
-/**
- * unify_create - create a file in global namespace first, so other 
- *    clients can see them. Create the file in storage nodes in background.
- */
-int32_t
-unify_create (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flags,
-	      mode_t mode,
-	      fd_t *fd)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->flags = flags;
-	local->fd = fd;
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, fd, loc->inode, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_create_cbk,
-		    NS(this),
-		    NS(this)->fops->create,
-		    loc,
-		    flags | O_EXCL,
-		    mode,
-		    fd);
-  
-	return 0;
-}
-
-
-/**
- * unify_opendir_cbk - 
- */
-int32_t
-unify_opendir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-	return 0;
-}
-
-/** 
- * unify_opendir -
- */
-int32_t
-unify_opendir (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       fd_t *fd)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame, unify_opendir_cbk,
-		    NS(this), NS(this)->fops->opendir, loc, fd);
-
-	return 0;
-}
-
-
-int32_t
-unify_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                   struct iatt *statpost)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = statpost->ia_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (IA_ISDIR (statpost->ia_type) || 
-				    !local->stpost.ia_blksize) {
-					local->stpre = *statpre;
-                                        local->stpost = *statpost;
-				}
-			}
-
-			if ((!IA_ISDIR (statpost->ia_type)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stpre = *statpre;
-                                local->stpost = *statpost;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->ia_ino)
-			local->op_ret = -1;
-
-                local->stpre.ia_ino = local->ia_ino;
-		local->stpost.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stpre, &local->stpost);
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-        if (!(loc && loc->inode)) {                     
-                STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
-                return 0;
-        }                                                    
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = 1;
-      
-                STACK_WIND (frame,
-                            unify_setattr_cbk,
-                            NS (this),
-                            NS (this)->fops->setattr,
-                            loc, stbuf, valid);
-        } else {
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		for (index = 0; local->list[index] != -1; index++) {
-			STACK_WIND (frame,
-				    unify_setattr_cbk,
-				    priv->xl_array[local->list[index]],
-				    priv->xl_array[local->list[index]]->fops->setattr,
-				    loc, stbuf, valid);
-
-			if (!--callcnt)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_setattr_cbk, child,
-			    child->fops->fsetattr, fd, stbuf, valid);
-
-		STACK_WIND (frame, unify_setattr_cbk, NS(this),
-			    NS(this)->fops->fsetattr, fd, stbuf, valid);
-	} else {
-		local->call_count = 1;
-    
-		STACK_WIND (frame, unify_setattr_cbk,
-			    NS(this), NS(this)->fops->fsetattr,
-			    fd, stbuf, valid);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_truncate_cbk - 
- */
-int32_t
-unify_truncate_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *prebuf,
-                    struct iatt *postbuf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			if (!((op_errno == ENOENT) && priv->optimist))
-				local->op_ret = -1;
-		}
-
-		if (op_ret >= 0) {
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = postbuf->ia_ino;
-				/* If the entry is directory, get the 
-				   stat from NS node */
-				if (IA_ISDIR (postbuf->ia_type) ||
-				    !local->stbuf.ia_blksize) {
-					local->stbuf = *prebuf;
-                                        local->poststbuf = *postbuf;
-				}
-			}
-
-			if ((!IA_ISDIR (postbuf->ia_type)) &&
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from 
-				   Storage node. */
-				local->stbuf = *prebuf;
-                                local->poststbuf = *postbuf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		if (local->ia_ino) {
-			local->stbuf.ia_ino = local->ia_ino;
-                        local->poststbuf.ia_ino = local->ia_ino;
-                } else {
-			local->op_ret = -1;
-                }
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf, &local->poststbuf);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_truncate - 
- */
-int32_t
-unify_truncate (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		off_t offset)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->ia_ino = loc->inode->ino;
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = 1;
-      
-		STACK_WIND (frame,
-			    unify_truncate_cbk,
-			    NS(this),
-			    NS(this)->fops->truncate,
-			    loc,
-                            0);
-	} else {
-		local->op_ret = 0;
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		/* Don't send offset to NS truncate */
-		STACK_WIND (frame, unify_truncate_cbk, NS(this),
-			    NS(this)->fops->truncate, loc, 0);
-		callcnt--;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[local->list[index]]) {
-				STACK_WIND (frame,
-					    unify_truncate_cbk,
-					    priv->xl_array[local->list[index]],
-					    priv->xl_array[local->list[index]]->fops->truncate,
-					    loc,
-					    offset);
-				if (!--callcnt)
-					break;
-			}
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_readlink_cbk - 
- */
-int32_t
-unify_readlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    const char *path,
-                    struct iatt *sbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
-	return 0;
-}
-
-/**
- * unify_readlink - Read the link only from the storage node.
- */
-int32_t
-unify_readlink (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		size_t size)
-{
-	unify_private_t *priv = this->private;
-	int32_t entry_count = 0;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-  
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		entry_count++;
-
-	if (entry_count >= 2) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_readlink_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->readlink,
-					    loc,
-					    size);
-				break;
-			}
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"returning ENOENT, no softlink files found "
-			"on storage node");
-		STACK_UNWIND (frame, -1, ENOENT, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink_cbk - 
- */
-int32_t
-unify_unlink_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0  || ((op_errno == ENOENT) && priv->optimist))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-
-                if (((call_frame_t *)cookie)->this == NS(this)) {
-                        local->oldpreparent = *preparent;
-                        local->oldpostparent = *postparent;
-                }
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink - 
- */
-int32_t
-unify_unlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		local->call_count++;
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_unlink_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->unlink,
-				    loc);
-			if (need_break)
-				break;
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: returning ENOENT", loc->path);
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_readv_cbk - 
- */
-int32_t
-unify_readv_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iovec *vector,
-		 int32_t count,
-		 struct iatt *stbuf,
-                 struct iobref *iobref)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf, iobref);
-	return 0;
-}
-
-/**
- * unify_readv - 
- */
-int32_t
-unify_readv (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t offset)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_readv_cbk,
-		    child,
-		    child->fops->readv,
-		    fd,
-		    size,
-		    offset);
-
-
-	return 0;
-}
-
-/**
- * unify_writev_cbk - 
- */
-int32_t
-unify_writev_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *prebuf,
-		  struct iatt *postbuf)
-{
-	unify_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->stbuf = *prebuf;
-        local->stbuf.ia_ino = local->ia_ino;
-
-        local->poststbuf = *postbuf;
-        local->poststbuf.ia_ino = local->ia_ino;
-
-	STACK_UNWIND (frame, op_ret, op_errno,
-                      &local->stbuf, &local->poststbuf);
-	return 0;
-}
-
-/**
- * unify_writev - 
- */
-int32_t
-unify_writev (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      struct iovec *vector,
-	      int32_t count,
-	      off_t off,
-              struct iobref *iobref)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-	unify_local_t *local = NULL;
-
-	INIT_LOCAL (frame, local);
-        local->ia_ino = fd->inode->ino;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_writev_cbk,
-		    child,
-		    child->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off,
-                    iobref);
-
-	return 0;
-}
-
-/**
- * unify_ftruncate -
- */
-int32_t
-unify_ftruncate (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 off_t offset)
-{
-	xlator_t *child = NULL;
-	unify_local_t *local = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->op_ret = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	local->call_count = 2;
-  
-	STACK_WIND (frame, unify_truncate_cbk,
-		    child, child->fops->ftruncate,
-		    fd, offset);
-  
-	STACK_WIND (frame, unify_truncate_cbk,
-		    NS(this), NS(this)->fops->ftruncate,
-		    fd, 0);
-  
-	return 0;
-}
-
-
-/**
- * unify_flush_cbk - 
- */
-int32_t
-unify_flush_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_flush -
- */
-int32_t
-unify_flush (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_flush_cbk, child, 
-		    child->fops->flush, fd);
-
-	return 0;
-}
-
-
-/**
- * unify_fsync_cbk - 
- */
-int32_t
-unify_fsync_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-                 struct iatt *prebuf,
-                 struct iatt *postbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-/**
- * unify_fsync - 
- */
-int32_t
-unify_fsync (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fsync_cbk, child,
-		    child->fops->fsync, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_fstat - Send fstat FOP to Namespace only if its directory, and to 
- *     both namespace and the storage node if its a file.
- */
-int32_t
-unify_fstat (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	INIT_LOCAL (frame, local);
-	local->ia_ino = fd->inode->ino;
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_buf_cbk, child,
-			    child->fops->fstat, fd);
-
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-
-	} else {
-		/* this is an directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-	}
-
-	return 0;
-}
-
-/**
- * unify_getdents_cbk - 
- */
-int32_t
-unify_getdents_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dir_entry_t *entry,
-		    int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entry, count);
-	return 0;
-}
-
-/**
- * unify_getdents - send the FOP request to all the nodes.
- */
-int32_t
-unify_getdents (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		size_t size,
-		off_t offset,
-		int32_t flag)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_getdents_cbk, NS(this),
-		    NS(this)->fops->getdents, fd, size, offset, flag);
-
-	return 0;
-}
-
-
-/**
- * unify_readdir_cbk - 
- */
-int32_t
-unify_readdir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-/**
- * unify_readdir - send the FOP request to all the nodes.
- */
-int32_t
-unify_readdir (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       size_t size,
-	       off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdir_cbk, NS(this),
-		    NS(this)->fops->readdir, fd, size, offset);
-
-	return 0;
-}
-
-
-int32_t
-unify_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-
-int32_t
-unify_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdirp_cbk, NS(this),
-		    NS(this)->fops->readdirp, fd, size, offset);
-
-	return 0;
-}
-
-
-/**
- * unify_fsyncdir_cbk - 
- */
-int32_t
-unify_fsyncdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-
-	return 0;
-}
-
-/**
- * unify_fsyncdir -
- */
-int32_t
-unify_fsyncdir (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		int32_t flags)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_fsyncdir_cbk,
-		    NS(this), NS(this)->fops->fsyncdir, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_lk_cbk - UNWIND frame with the proper return arguments.
- */
-int32_t
-unify_lk_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct gf_flock *lock)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, lock);
-	return 0;
-}
-
-/**
- * unify_lk - Send it to all the storage nodes, (should be 1) which has file.
- */
-int32_t
-unify_lk (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  int32_t cmd,
-	  struct gf_flock *lock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_lk_cbk, child,
-		    child->fops->lk, fd, cmd, lock);
-
-	return 0;
-}
-
-
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-static int32_t
-unify_setxattr_file_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	unify_private_t *private = this->private;
-	unify_local_t *local = frame->local;
-	xlator_t *sched_xl = NULL;
-	struct sched_ops *sched_ops = NULL;
-
-	if (op_ret == -1) {
-		if (!ENOTSUP)
-			gf_log (this->name, GF_LOG_ERROR,
-				"setxattr with XATTR_CREATE on ns: "
-				"path(%s) key(%s): %s",
-				local->loc1.path, local->name, 
-				strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno);
-		return 0;
-	} 
-
-	LOCK (&frame->lock);
-	{
-		local->failed = 0;
-		local->op_ret = 0;
-		local->op_errno = 0;
-		local->call_count = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	/* schedule XATTR_CREATE on one of the child node */
-	sched_ops = private->sched_ops;
-    
-	/* Send create request to the scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->name); 
-	if (!sched_xl) {
-		STACK_UNWIND (frame, -1, ENOTCONN);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_setxattr_cbk,
-		    sched_xl,
-		    sched_xl->fops->setxattr,
-		    &local->loc1,
-		    local->dict,
-		    local->flags);
-	return 0;
-}
-
-/**
- * unify_setxattr_cbk - When all the child nodes return, UNWIND frame.
- */
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	dict_t *dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, (((op_errno == ENOENT) || 
-					      (op_errno == ENOTSUP))? 
-					     GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			if (local->failed == -1) {
-				local->failed = 1;
-			}
-			local->op_errno = op_errno;
-		} else {
-			local->failed = 0;
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed && local->name && 
-		    ZR_FILE_CONTENT_REQUEST(local->name)) {      
-			dict = get_new_dict ();
-			dict_set (dict, local->dict->members_list->key, 
-				  data_from_dynptr(NULL, 0));
-			dict_ref (dict);
-
-			local->call_count = 1;
-
-			STACK_WIND (frame,
-				    unify_setxattr_file_cbk,
-				    NS(this),
-				    NS(this)->fops->setxattr,
-				    &local->loc1,
-				    dict,
-				    XATTR_CREATE);
-
-			dict_unref (dict);
-			return 0;
-		}
-    
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sexattr - This function should be sent to all the storage nodes, 
- *       which contains the file, (excluding namespace).
- */
-int32_t
-unify_setxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *dict,
-		int32_t flags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-	data_pair_t *trav = dict->members_list;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->failed = -1;
-	loc_copy (&local->loc1, loc);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-
-		if (trav && trav->key && ZR_FILE_CONTENT_REQUEST(trav->key)) {
-			/* direct the storage xlators to change file 
-			   content only if file exists */
-			local->flags = flags;
-			local->dict = dict;
-			local->name = gf_strdup (trav->key);
-			flags |= XATTR_REPLACE;
-		}
-
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_setxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->setxattr,
-				    loc, dict, flags);
-		}
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-  
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_setxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->setxattr,
-					    loc,
-					    dict,
-					    flags);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	}
-
-	/* No entry in storage nodes */
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"returning ENOENT, file not found on storage node.");
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-/**
- * unify_getxattr_cbk - This function is called from only one child, so, no
- *     need of any lock or anything else, just send it to above layer 
- */
-int32_t
-unify_getxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dict_t *value)
-{
-	int32_t callcnt = 0;
-	dict_t *local_value = NULL;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, 
-				(((op_errno == ENOENT) || 
-				  (op_errno == ENODATA) || 
-				  (op_errno == ENOTSUP)) ? 
-				 GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-		} else {
-			if (!local->dict)
-				local->dict = dict_ref (value);
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local_value = local->dict;
-		local->dict = NULL;
-      
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local_value);
-      
-		if (local_value)
-			dict_unref (local_value);
-	} 
-
-	return 0;
-}
-
-
-/** 
- * unify_getxattr - This FOP is sent to only the storage node.
- */
-int32_t
-unify_getxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		const char *name)
-{
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t count = 0;
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	INIT_LOCAL (frame, local);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame,
-				    unify_getxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->getxattr,
-				    loc,
-				    name);
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			count++;
-		}
-	}
-
-	if (count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_getxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->getxattr,
-					    loc,
-					    name);
-				if (!--count)
-					break;
-			}
-		}
-	} else {
-		dict_t *tmp_dict = get_new_dict ();
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s: returning ENODATA, no file found on storage node",
-			loc->path);
-		STACK_UNWIND (frame, -1, ENODATA, tmp_dict);
-		dict_destroy (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr_cbk - Wait till all the child node returns the call
- *      and then UNWIND to above layer.
- */
-int32_t
-unify_removexattr_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{ 
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			if (op_errno != ENOTSUP)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): path(%s): %s", 
-					prev_frame->this->name, 
-					local->loc1.path, strerror (op_errno));
-		} else {
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);  
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr - Send it to all the child nodes which has the files.
- */
-int32_t
-unify_removexattr (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc,
-		   const char *name)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame, 		    
-				    unify_removexattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->removexattr,
-				    loc,
-				    name);
-
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_removexattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->removexattr,
-					    loc,
-					    name);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	} 
-
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"%s: returning ENOENT, not found on storage node.", loc->path);
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-int32_t 
-unify_mknod_unlink_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-                        struct iatt *preparent,
-                        struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-  
-	unify_local_wipe (local);
-	/* No log required here as this -1 is for mknod call */
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_mknod_cbk - 
- */
-int32_t
-unify_mknod_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, sending unlink to "
-			"namespace");
-		local->op_errno = op_errno;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-	return 0;
-}
-
-/**
- * unify_ns_mknod_cbk - 
- */
-int32_t
-unify_ns_mknod_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	call_frame_t *prev_frame = cookie;
-
-	if (op_ret == -1) {
-		/* No need to send mknod request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s): %s", 
-			prev_frame->this->name, local->loc1.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                              preparent, postparent);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	list = GF_CALLOC (1, sizeof (int16_t) * 3, gf_unify_mt_int16_t);
-	ERR_ABORT (list);
-	list[0] = priv->child_count;
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send mknod request to scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, no node online "
-			"at the moment, sending unlink to NS");
-		local->op_errno = ENOTCONN;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-  
-	STACK_WIND (frame,  unify_mknod_cbk,
-		    sched_xl,  sched_xl->fops->mknod,
-		    &local->loc1, local->mode, local->dev);
-
-	return 0;
-}
-
-/**
- * unify_mknod - Create a device on namespace first, and later create on 
- *       the storage node.
- */
-int32_t
-unify_mknod (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode,
-	     dev_t rdev)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->dev = rdev;
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mknod_cbk,
-		    NS(this),
-		    NS(this)->fops->mknod,
-		    loc,
-		    mode,
-		    rdev);
-
-	return 0;
-}
-
-int32_t 
-unify_symlink_unlink_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-                          struct iatt *preparent,
-                          struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_symlink_cbk - 
- */
-int32_t
-unify_symlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, sending unlink "
-			"to namespace");
-
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-    
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_symlink_cbk - 
- */
-int32_t
-unify_ns_symlink_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-                      struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent)
-{
-
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	int16_t *list = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send symlink request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, buf,
-                              preparent, postparent);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->ia_ino = buf->ia_ino;
-  
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	/* Start the mapping list */
-
-	list = GF_CALLOC (1, sizeof (int16_t) * 3, gf_unify_mt_int16_t);
-	ERR_ABORT (list);
-	list[0] = priv->child_count; //namespace's index
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send symlink request to all the nodes now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, no node online, "
-			"sending unlink to namespace");
-      
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-	
-	STACK_WIND (frame,
-		    unify_symlink_cbk,
-		    sched_xl,
-		    sched_xl->fops->symlink,
-		    local->name,
-		    &local->loc1);
-
-	return 0;
-}
-
-/**
- * unify_symlink - 
- */
-int32_t
-unify_symlink (call_frame_t *frame,
-	       xlator_t *this,
-	       const char *linkpath,
-	       loc_t *loc)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->name = gf_strdup (linkpath);
-
-	if ((local->name == NULL) || 
-	    (local->loc1.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_symlink_cbk,
-		    NS(this),
-		    NS(this)->fops->symlink,
-		    linkpath,
-		    loc);
-
-	return 0;
-}
-
-
-int32_t 
-unify_rename_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s -> %s): %s", 
-			prev_frame->this->name, 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-      
-	}
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_undo_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  struct iatt *buf,
-                          struct iatt *preoldparent,
-                          struct iatt *postoldparent,
-                          struct iatt *prenewparent,
-                          struct iatt *postnewparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-	}
-
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, &local->stbuf);
-	return 0;
-}
-
-int32_t 
-unify_rename_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *buf,
-                  struct iatt *preoldparent,
-                  struct iatt *postoldparent,
-                  struct iatt *prenewparent,
-                  struct iatt *postnewparent)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (!IA_ISDIR (buf->ia_type))
-				local->stbuf = *buf;
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"child(%s): path(%s -> %s): %s", 
-				prev_frame->this->name, 
-				local->loc1.path, local->loc2.path, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.ia_ino = local->ia_ino;
-		if (IA_ISDIR (local->loc1.inode->ia_type)) {
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                                      &local->stbuf, &local->oldpreparent,
-                                      &local->oldpostparent, &local->newpreparent,
-                                      &local->newpostparent);
-			return 0;
-		}
-
-		if (local->op_ret == -1) {
-			/* TODO: check this logic */
-
-			/* Rename failed in storage node, successful on NS, 
-			 * hence, rename back the entries in NS */
-			/* NOTE: this will be done only if the destination 
-			 * doesn't exists, if  the destination exists, the 
-			 * job of correcting NS is left to self-heal
-			 */
-			if (!local->index) {
-				loc_t tmp_oldloc = {
-                                        /* its actual 'newloc->path' */
-					.path = local->loc2.path, 
-					.inode = local->loc1.inode,
-					.parent = local->loc2.parent
-				};
-	
-				loc_t tmp_newloc = {
-					/* Actual 'oldloc->path' */
-					.path = local->loc1.path,
-					.parent = local->loc1.parent
-				};
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"rename succussful on namespace, on "
-					"stroage node failed, reverting back");
-
-				STACK_WIND (frame,
-					    unify_ns_rename_undo_cbk,
-					    NS(this),
-					    NS(this)->fops->rename,
-					    &tmp_oldloc,
-					    &tmp_newloc);
-				return 0;
-			}
-		} else {
-			/* Rename successful on storage nodes */
-
-			int32_t idx = 0;
-			int16_t *tmp_list = NULL;
-			uint64_t tmp_list_int64 = 0;
-			if (local->loc2.inode) {
-				inode_ctx_get (local->loc2.inode, 
-					       this, &tmp_list_int64);
-				list = (int16_t *)(long)tmp_list_int64;
-
-			}
-
-			if (list) {				
-				for (index = 0; list[index] != -1; index++);
-				tmp_list = GF_CALLOC (1, index * 2, 
-                                                      gf_unify_mt_int16_t);
-				memcpy (tmp_list, list, index * 2);
-
-				for (index = 0; list[index] != -1; index++) {
-					/* TODO: Check this logic. */
-					/* If the destination file exists in 
-					 * the same storage node where we sent
-					 * 'rename' call, no need to send 
-					 * unlink 
-					 */
-					for (idx = 0; 
-					     local->list[idx] != -1; idx++) {
-						if (tmp_list[index] == local->list[idx]) {
-							tmp_list[index] = priv->child_count;
-							continue;
-						}
-					}
-	  
-					if (NS(this) != priv->xl_array[tmp_list[index]]) {
-						local->call_count++;
-						callcnt++;
-					}
-				}
-
-				if (local->call_count) {
-					if (callcnt > 1)
-						gf_log (this->name, 
-							GF_LOG_ERROR, 
-							"%s->%s: more (%d) "
-							"subvolumes have the "
-							"newloc entry", 
-							local->loc1.path, 
-							local->loc2.path, 
-							callcnt);
-
-					for (index=0; 
-					     tmp_list[index] != -1; index++) {
-						if (NS(this) != priv->xl_array[tmp_list[index]]) {		    
-							STACK_WIND (frame,
-								    unify_rename_unlink_cbk,
-								    priv->xl_array[tmp_list[index]],
-								    priv->xl_array[tmp_list[index]]->fops->unlink,
-								    &local->loc2);
-							if (!--callcnt)
-								break;
-						}
-					}
-
-					GF_FREE (tmp_list);
-					return 0;
-				}
-				if (tmp_list)
-					GF_FREE (tmp_list);
-			}
-		}
-    
-		/* Need not send 'unlink' to storage node */
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent,
-                              &local->newpreparent, &local->newpostparent);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Free local->new_inode */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, buf,
-                              preoldparent, postoldparent,
-                              prenewparent, postnewparent);
-		return 0;
-	}
-
-	local->stbuf = *buf;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preoldparent;
-        local->oldpostparent = *postoldparent;
-        local->newpreparent = *prenewparent;
-        local->newpostparent = *postnewparent;
-
-	/* Everything is fine. */
-	if (IA_ISDIR (buf->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index=0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_rename_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->rename,
-				    &local->loc1,
-				    &local->loc2);
-		}
-
-		return 0;
-	}
-
-	local->call_count = 0;  
-	/* send rename */
-	list = local->list;
-	for (index=0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			callcnt++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index=0; list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[list[index]]) {
-				STACK_WIND (frame,
-					    unify_rename_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->rename,
-					    &local->loc1,
-					    &local->loc2);
-				if (!--callcnt)
-					break;
-			}
-		}
-	} else {
-		/* file doesn't seem to be present in storage nodes */
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"CRITICAL: source file not in storage node, "
-			"rename successful on namespace :O");
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, -1, EIO, NULL,
-                              NULL, NULL, /* preoldparent, postoldparent */
-                              NULL, NULL); /* prenewparent, postnewparent */
-	}
-	return 0;
-}
-
-
-/**
- * unify_rename - One of the tricky function. The deadliest of all :O
- */
-int32_t
-unify_rename (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	if ((local->loc1.path == NULL) || 
-	    (local->loc2.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL,
-                              NULL, NULL, /* preoldparent, postoldparent */
-                              NULL, NULL); /* prenewparent, postnewparent */
-		return 0;
-	}
-  
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_rename_cbk,
-		    NS(this),
-		    NS(this)->fops->rename,
-		    oldloc,
-		    newloc);
-	return 0;
-}
-
-/**
- * unify_link_cbk -
- */
-int32_t
-unify_link_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret >= 0) 
-		local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_link_cbk - 
- */
-int32_t
-unify_ns_link_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	int16_t *list = local->list;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send link request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                              preparent, postparent);
-		return 0;
-	}
-
-	/* Update inode for this entry */
-	local->op_ret = 0;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	/* Send link request to the node now */
-	for (index = 0; list[index] != -1; index++) {
-		char need_break = (list[index+1] == -1);
-		if (priv->xl_array[list[index]] != NS (this)) {
-			STACK_WIND (frame,
-				    unify_link_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->link,
-				    &local->loc1,
-				    &local->loc2);
-                        break;
-		}
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-/**
- * unify_link - 
- */
-int32_t
-unify_link (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *oldloc,
-	    loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (oldloc);
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (newloc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_link_cbk,
-		    NS(this),
-		    NS(this)->fops->link,
-		    oldloc,
-		    newloc);
-
-	return 0;
-}
-
-
-/**
- * unify_checksum_cbk - 
- */
-int32_t
-unify_checksum_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    uint8_t *fchecksum,
-		    uint8_t *dchecksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fchecksum, dchecksum);
-
-	return 0;
-}
-
-/**
- * unify_checksum - 
- */
-int32_t
-unify_checksum (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t flag)
-{
-	STACK_WIND (frame,
-		    unify_checksum_cbk,
-		    NS(this),
-		    NS(this)->fops->checksum,
-		    loc,
-		    flag);
-
-	return 0;
-}
-
-
-/**
- * unify_finodelk_cbk - 
- */
-int
-unify_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_finodelk
- */
-int
-unify_finodelk (call_frame_t *frame, xlator_t *this,
-		const char *volume, fd_t *fd, int cmd, struct gf_flock *flock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_finodelk_cbk,
-		    child, child->fops->finodelk,
-		    volume, fd, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fentrylk_cbk - 
- */
-int
-unify_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_fentrylk
- */
-int
-unify_fentrylk (call_frame_t *frame, xlator_t *this,
-		const char *volume, fd_t *fd, const char *basename,
-		entrylk_cmd cmd, entrylk_type type)
-		
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fentrylk_cbk,
-		    child, child->fops->fentrylk,
-		    volume, fd, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fxattrop_cbk - 
- */
-int
-unify_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_fxattrop
- */
-int
-unify_fxattrop (call_frame_t *frame, xlator_t *this,
-		fd_t *fd, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fxattrop_cbk,
-		    child, child->fops->fxattrop,
-		    fd, optype, xattr);
-
-	return 0;
-}
-
-
-/**
- * unify_inodelk_cbk - 
- */
-int
-unify_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_inodelk
- */
-int
-unify_inodelk (call_frame_t *frame, xlator_t *this,
-	       const char *volume, loc_t *loc, int cmd, struct gf_flock *flock)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_inodelk_cbk,
-		    child, child->fops->inodelk,
-		    volume, loc, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_entrylk_cbk - 
- */
-int
-unify_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_entrylk
- */
-int
-unify_entrylk (call_frame_t *frame, xlator_t *this,
-	       const char *volume, loc_t *loc, const char *basename,
-	       entrylk_cmd cmd, entrylk_type type)
-		
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_entrylk_cbk,
-		    child, child->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_xattrop_cbk - 
- */
-int
-unify_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_xattrop
- */
-int
-unify_xattrop (call_frame_t *frame, xlator_t *this,
-		loc_t *loc, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_xattrop_cbk,
-		    child, child->fops->xattrop,
-		    loc, optype, xattr);
-
-	return 0;
-}
-
-int
-unify_forget (xlator_t *this,
-              inode_t *inode)
-{
-        int16_t *list = NULL;
-        uint64_t tmp_list = 0;
-
-        if (inode->ia_type && (!IA_ISDIR(inode->ia_type))) {
-                inode_ctx_get (inode, this, &tmp_list);
-                if (tmp_list) {
-                        list = (int16_t *)(long)tmp_list;
-                        GF_FREE (list);
-                }
-        }
-
-        return 0;
-}
-
-/**
- * notify
- */
-int32_t
-notify (xlator_t *this,
-        int32_t event,
-        void *data,
-        ...)
-{
-	unify_private_t *priv = this->private;
-	struct sched_ops *sched = NULL;
-
-	if (!priv) {
-		return 0;
-	}
-
-	sched = priv->sched_ops;    
-	if (!sched) {
-		gf_log (this->name, GF_LOG_CRITICAL, "No scheduler :O");
-		raise (SIGTERM);
-		return 0;
-	}
-	if (priv->namespace == data) {
-		if (event == GF_EVENT_CHILD_UP) {
-			sched->notify (this, event, data);
-		}
-		return 0;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		/* Call scheduler's update () to enable it for scheduling */
-		sched->notify (this, event, data);
-	
-		LOCK (&priv->lock);
-		{
-			/* Increment the inode's generation, which is 
-			   used for self_heal */
-			++priv->inode_generation;
-			++priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (!priv->is_up) {
-			default_notify (this, event, data);
-			priv->is_up = 1;
-		}
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		/* Call scheduler's update () to disable the child node 
-		 * for scheduling
-		 */
-		sched->notify (this, event, data);
-		LOCK (&priv->lock);
-		{
-			--priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (priv->num_child_up == 0) {
-			/* Send CHILD_DOWN to upper layer */
-			default_notify (this, event, data);
-			priv->is_up = 0;
-		}
-	}
-	break;
-
-	default:
-	{
-		default_notify (this, event, data);
-	}
-	break;
-	}
-
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_unify_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-/** 
- * init - This function is called first in the xlator, while initializing.
- *   All the config file options are checked and appropriate flags are set.
- *
- * @this - 
- */
-int32_t 
-init (xlator_t *this)
-{
-	int32_t          ret       = 0;
-	int32_t          count     = 0;
-	data_t          *scheduler = NULL;
-	data_t          *data      = NULL;
-	xlator_t        *ns_xl     = NULL;
-	xlator_list_t   *trav      = NULL;
-	xlator_list_t   *xlparent  = NULL;
-	xlator_list_t   *parent    = NULL;
-	unify_private_t *_private  = NULL; 
-
-
-	/* Check for number of child nodes, if there is no child nodes, exit */
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"No child nodes specified. check \"subvolumes \" "
-			"option in volfile");
-		return -1;
-	}
-
-  	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	/* Check for 'scheduler' in volume */
-	scheduler = dict_get (this->options, "scheduler");
-	if (!scheduler) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"\"option scheduler <x>\" is missing in volfile");
-		return -1;
-	}
-
-	/* Setting "option namespace <node>" */
-	data = dict_get (this->options, "namespace");
-	if(!data) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace option not specified, Exiting");
-		return -1;
-	}
-	/* Search namespace in the child node, if found, exit */
-	trav = this->children;
-	while (trav) {
-		if (strcmp (trav->xlator->name, data->data) == 0)
-			break;
-		trav = trav->next;
-	}
-	if (trav) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node used as a subvolume, Exiting");
-		return -1;
-	}
-	
-	/* Search for the namespace node, if found, continue */
-	ns_xl = this->next;
-	while (ns_xl) {
-		if (strcmp (ns_xl->name, data->data) == 0)
-			break;
-		ns_xl = ns_xl->next;
-	}
-	if (!ns_xl) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node not found in volfile, Exiting");
-		return -1;
-	}
-	
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"namespace node specified as %s", data->data);
-	
-	_private = GF_CALLOC (1, sizeof (*_private), 
-                              gf_unify_mt_unify_private_t);
-	ERR_ABORT (_private);
-	_private->sched_ops = get_scheduler (this, scheduler->data);
-	if (!_private->sched_ops) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"Error while loading scheduler. Exiting");
-		GF_FREE (_private);
-		return -1;
-	}
-	
-	if (ns_xl->parents) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Namespace node should not be a child of any other node. Exiting");
-		GF_FREE (_private);
-		return -1;
-	}
-
-	_private->namespace = ns_xl;
-	
-	/* update _private structure */
-	{
-		count = 0;
-		trav = this->children;
-		/* Get the number of child count */
-		while (trav) {
-			count++;
-			trav = trav->next;
-		}
-		
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"Child node count is %d", count);    
-
-		_private->child_count = count;
-		if (count == 1) {
-			/* TODO: Should I error out here? */
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"WARNING: You have defined only one "
-				"\"subvolumes\" for unify volume. It may not "
-				"be the desired config, review your volume "
-				"volfile. If this is how you are testing it,"
-				" you may hit some performance penalty");
-		}
-		
-		_private->xl_array = GF_CALLOC (1, 
-					        sizeof (xlator_t) * (count + 1),
-                                                gf_unify_mt_xlator_t);
-		ERR_ABORT (_private->xl_array);
-		
-		count = 0;
-		trav = this->children;
-		while (trav) {
-			_private->xl_array[count++] = trav->xlator;
-			trav = trav->next;
-		}
-		_private->xl_array[count] = _private->namespace;
-		
-		/* self-heal part, start with generation '1' */
-		_private->inode_generation = 1; 
-                /* Because, Foreground part is tested well */
-		_private->self_heal = ZR_UNIFY_FG_SELF_HEAL; 
-		data = dict_get (this->options, "self-heal");
-		if (data) {
-			if (strcasecmp (data->data, "off") == 0) 
-				_private->self_heal = ZR_UNIFY_SELF_HEAL_OFF;
-
-			if (strcasecmp (data->data, "foreground") == 0)
-				_private->self_heal = ZR_UNIFY_FG_SELF_HEAL;
-
-			if (strcasecmp (data->data, "background") == 0)
-				_private->self_heal = ZR_UNIFY_BG_SELF_HEAL;
-		}
-    
-		/* optimist - ask bulde for more about it */
-		data = dict_get (this->options, "optimist");
-		if (data) {
-			if (gf_string2boolean (data->data, 
-					       &_private->optimist) == -1) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"optimist excepts only boolean "
-					"options");
-			}
-		}
-
-		LOCK_INIT (&_private->lock);
-	}
-
-	/* Now that everything is fine. */
-	this->private = (void *)_private;
-	{
-                ret = _private->sched_ops->mem_acct_init (this);
-
-                if (ret == -1) {
-                        return -1;
-                }
-
-		/* Initialize scheduler, if everything else is successful */
-		ret = _private->sched_ops->init (this); 
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_CRITICAL,
-				"Initializing scheduler failed, Exiting");
-			GF_FREE (_private);
-			return -1;
-		}
-
-
-		ret = 0;
-
-		/* This section is required because some fops may look 
-		 * for 'xl->parent' variable 
-		 */
-		xlparent = GF_CALLOC (1, sizeof (*xlparent),
-                                      gf_unify_mt_xlator_list_t);
-		xlparent->xlator = this;
-		if (!ns_xl->parents) {
-			ns_xl->parents = xlparent;
-		} else {
-			parent = ns_xl->parents;
-			while (parent->next)
-				parent = parent->next;
-			parent->next = xlparent;
-		}
-	}
-
-	/* Tell namespace node that init is done */
-	xlator_notify (ns_xl, GF_EVENT_PARENT_UP, this);
-
-	return 0;
-}
-
-/** 
- * fini  - Free all the allocated memory 
- */
-void
-fini (xlator_t *this)
-{
-	unify_private_t *priv = this->private;
-	priv->sched_ops->fini (this);
-	this->private = NULL;
-	LOCK_DESTROY (&priv->lock);
-	GF_FREE (priv->xl_array);
-	GF_FREE (priv);
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.stat        = unify_stat,
-	.readlink    = unify_readlink,
-	.mknod       = unify_mknod,
-	.mkdir       = unify_mkdir,
-	.unlink      = unify_unlink,
-	.rmdir       = unify_rmdir,
-	.symlink     = unify_symlink,
-	.rename      = unify_rename,
-	.link        = unify_link,
-	.truncate    = unify_truncate,
-	.create      = unify_create,
-	.open        = unify_open,
-	.readv       = unify_readv,
-	.writev      = unify_writev,
-	.statfs      = unify_statfs,
-	.flush       = unify_flush,
-	.fsync       = unify_fsync,
-	.setxattr    = unify_setxattr,
-	.getxattr    = unify_getxattr,
-	.removexattr = unify_removexattr,
-	.opendir     = unify_opendir,
-	.readdir     = unify_readdir,
-	.readdirp    = unify_readdirp,
-	.fsyncdir    = unify_fsyncdir,
-	.access      = unify_access,
-	.ftruncate   = unify_ftruncate,
-	.fstat       = unify_fstat,
-	.lk          = unify_lk,
-	.lookup      = unify_lookup,
-	.getdents    = unify_getdents,
-	.checksum    = unify_checksum,
-	.inodelk     = unify_inodelk,
-	.finodelk    = unify_finodelk,
-	.entrylk     = unify_entrylk,
-	.fentrylk    = unify_fentrylk,
-	.xattrop     = unify_xattrop,
-	.fxattrop    = unify_fxattrop,
-        .setattr     = unify_setattr, 
-        .fsetattr    = unify_fsetattr,
-};
-
-
-struct xlator_cbks cbks = {
-        .forget  = unify_forget,
-};
-
-struct volume_options options[] = {
-	{ .key   = { "namespace" },  
-	  .type  = GF_OPTION_TYPE_XLATOR 
-	},
-	{ .key   = { "scheduler" },  
-	  .value = { "alu", "rr", "random", "nufa", "switch" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	{ .key   = {"self-heal"},  
-	  .value = { "foreground", "background", "off" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	/* TODO: remove it some time later */
-	{ .key   = {"optimist"},  
-	  .type  = GF_OPTION_TYPE_BOOL 
-	},
-
-	{ .key   = {NULL} },
-};
diff --git a/xlators/cluster/unify/src/unify.h b/xlators/cluster/unify/src/unify.h
deleted file mode 100644
index dbd5e44a2..000000000
--- a/xlators/cluster/unify/src/unify.h
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _UNIFY_H
-#define _UNIFY_H
-
-#include "scheduler.h"
-#include "list.h"
-#include "unify-mem-types.h"
-
-#define MAX_DIR_ENTRY_STRING     (32 * 1024)
-
-#define ZR_UNIFY_SELF_HEAL_OFF 0
-#define ZR_UNIFY_FG_SELF_HEAL  1
-#define ZR_UNIFY_BG_SELF_HEAL  2
-
-/* Sometimes one should use completely random numbers.. its good :p */
-#define UNIFY_SELF_HEAL_GETDENTS_COUNT 512
-
-#define NS(xl)          (((unify_private_t *)xl->private)->namespace)
-
-/* This is used to allocate memory for local structure */
-#define INIT_LOCAL(fr, loc)                   \
-do {                                          \
-  loc = GF_CALLOC (1, sizeof (unify_local_t), gf_unify_mt_unify_local_t);   \
-  ERR_ABORT (loc);			      \
-  if (!loc) {                                 \
-    STACK_UNWIND (fr, -1, ENOMEM);            \
-    return 0;                                 \
-  }                                           \
-  fr->local = loc;                            \
-  loc->op_ret = -1;                           \
-  loc->op_errno = ENOENT;                     \
-} while (0)
-
-
-
-struct unify_private {
-	/* Update this structure depending on requirement */
-	void *scheduler;               /* THIS SHOULD BE THE FIRST VARIABLE, 
-					  if xlator is using scheduler */
-	struct sched_ops *sched_ops;   /* Scheduler options  */
-	xlator_t *namespace;           /* ptr to namespace xlator */
-	xlator_t **xl_array;
-	gf_boolean_t optimist;
-	int16_t child_count;
-	int16_t num_child_up;
-	uint8_t self_heal;
-	uint8_t is_up;
-	uint64_t inode_generation;
-	gf_lock_t lock;
-};
-typedef struct unify_private unify_private_t;
-
-struct unify_self_heal_struct {
-	uint8_t dir_checksum[NAME_MAX];
-	uint8_t ns_dir_checksum[NAME_MAX];
-	uint8_t file_checksum[NAME_MAX];
-	uint8_t ns_file_checksum[NAME_MAX];
-	off_t *offset_list;
-	int   *count_list;
-	dir_entry_t **entry_list;
-};
-
-
-struct _unify_local_t {
-	int32_t call_count;
-	int32_t op_ret;
-	int32_t op_errno;
-	mode_t mode;
-	off_t offset;
-	dev_t dev;
-	uid_t uid;
-	gid_t gid;
-	int32_t flags;
-	int32_t entry_count;
-	int32_t count;    // dir_entry_t count;
-	fd_t *fd;
-	struct iatt stbuf;
-        struct iatt stpre;
-        struct iatt stpost;
-	struct statvfs statvfs_buf;
-	struct timespec tv[2];
-	char *name;
-	int32_t revalidate;
-
-	ino_t ia_ino;
-	nlink_t ia_nlink;
-  
-	dict_t *dict;
-
-	int16_t *list;
-	int16_t *new_list; /* Used only in case of rename */
-	int16_t index;
-
-	int32_t failed;
-	int32_t return_eio;  /* Used in case of different st-mode 
-				present for a given path */
-
-	uint64_t inode_generation; /* used to store the per directory 
-				    * inode_generation. Got from inode's ctx 
-				    * of directory inodes
-				    */
-
-	struct unify_self_heal_struct *sh_struct;
-	loc_t loc1, loc2;
-
-        struct iatt poststbuf;
-        /* When not used for rename, old*
-         * are used as the attrs for the current
-         * parent directory.
-         */
-        struct iatt oldpreparent;
-        struct iatt oldpostparent;
-        struct iatt newpreparent;
-        struct iatt newpostparent;
-        int32_t wbflags;
-};
-typedef struct _unify_local_t unify_local_t;
-
-int32_t zr_unify_self_heal (call_frame_t *frame,
-			    xlator_t *this,
-			    unify_local_t *local);
-
-#endif /* _UNIFY_H */
diff --git a/xlators/debug/error-gen/src/Makefile.am b/xlators/debug/error-gen/src/Makefile.am
index df9080358..5075c59a8 100644
--- a/xlators/debug/error-gen/src/Makefile.am
+++ b/xlators/debug/error-gen/src/Makefile.am
@@ -2,15 +2,16 @@
 xlator_LTLIBRARIES = error-gen.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-error_gen_la_LDFLAGS = -module -avoidversion
+error_gen_la_LDFLAGS = -module -avoid-version
 
 error_gen_la_SOURCES = error-gen.c
 error_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = error-gen.h error-gen-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/error-gen/src/error-gen-mem-types.h b/xlators/debug/error-gen/src/error-gen-mem-types.h
index b643dc5f7..f02280535 100644
--- a/xlators/debug/error-gen/src/error-gen-mem-types.h
+++ b/xlators/debug/error-gen/src/error-gen-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __ERROR_GEN_MEM_TYPES_H__
 #define __ERROR_GEN_MEM_TYPES_H__
 
diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c
index e44baf5b7..ec0874b35 100644
--- a/xlators/debug/error-gen/src/error-gen.c
+++ b/xlators/debug/error-gen/src/error-gen.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -24,6 +14,7 @@
 
 #include "xlator.h"
 #include "error-gen.h"
+#include "statedump.h"
 
 sys_error_t error_no_list[] = {
         [GF_FOP_LOOKUP]            = { .error_no_count = 4,
@@ -91,9 +82,10 @@ sys_error_t error_no_list[] = {
         [GF_FOP_READ]              = { .error_no_count = 5,
                                     .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
                                                  ENAMETOOLONG}},
-        [GF_FOP_WRITE]             = { .error_no_count = 5,
+        [GF_FOP_WRITE]             = { .error_no_count = 7,
                                     .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG}},
+                                                 ENAMETOOLONG,ENOSPC,
+						 GF_ERROR_SHORT_WRITE}},
         [GF_FOP_STATFS]            = {.error_no_count = 10,
                                    .error_no = {EACCES,EBADF,EFAULT,EINTR,
                                                 EIO,ENAMETOOLONG,ENOENT,
@@ -246,6 +238,8 @@ conv_errno_to_int (char **error_no)
                 return EINTR;
         else if (!strcmp ((*error_no), "EFBIG"))
                 return EFBIG;
+	else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
+		return GF_ERROR_SHORT_WRITE;
         else
                 return EAGAIN;
 }
@@ -387,17 +381,17 @@ error_gen (xlator_t *this, int op_no)
 int
 error_gen_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno, inode_t *inode,
-		      struct iatt *buf, dict_t *dict, struct iatt *postparent)
+		      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
 	STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode,
-                             buf, dict, postparent);
-	return 0;
+                             buf, xdata, postparent);
+        return 0;
 }
 
 
 int
 error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  dict_t *xattr_req)
+                  dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -412,36 +406,28 @@ error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                              NULL);
-		return 0;
+                                     NULL);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_lookup_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->lookup,
-		    loc, xattr_req);
-	return 0;
-}
-
-
-int
-error_gen_forget (xlator_t *this, inode_t *inode)
-{
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, struct iatt *buf)
+		    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
-
-	return 0;
+	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 int
-error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -455,32 +441,31 @@ error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_stat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->stat,
-		    loc);
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop)
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
-
-	return 0;
+	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid)
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -494,21 +479,21 @@ error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->setattr,
-		    loc, stbuf, valid);
-	return 0;
+		    loc, stbuf, valid, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid)
+                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -522,32 +507,32 @@ error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsetattr,
-		    fd, stbuf, valid);
-	return 0;
+		    fd, stbuf, valid, xdata);
+        return 0;
 }
 
 
 int
 error_gen_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			int32_t op_ret, int32_t op_errno,
-			struct iatt *prebuf, struct iatt *postbuf)
+			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
-	return 0;
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset)
+                    off_t offset, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -562,32 +547,32 @@ error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (truncate, frame, -1, op_errno,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_truncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset);
-	return 0;
+		    loc, offset, xdata);
+        return 0;
 }
 
 
 int
 error_gen_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf)
+                         struct iatt *postbuf, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
-	return 0;
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     off_t offset)
+		     off_t offset, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp =NULL;
@@ -602,31 +587,30 @@ error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_ftruncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset);
-	return 0;
+		    fd, offset, xdata);
+        return 0;
 }
 
 
 int
 error_gen_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno)
+		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t mask)
+		  int32_t mask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -640,31 +624,31 @@ error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (access, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (access, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_access_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->access,
-		    loc, mask);
-	return 0;
+		    loc, mask, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			int32_t op_ret, int32_t op_errno,
-			const char *path, struct iatt *sbuf)
+			const char *path, struct iatt *sbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf);
-	return 0;
+	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    size_t size)
+                    size_t size, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -678,15 +662,15 @@ error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_readlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readlink,
-		    loc, size);
-	return 0;
+		    loc, size, xdata);
+        return 0;
 }
 
 
@@ -694,18 +678,18 @@ int
 error_gen_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno,
                              inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		 mode_t mode, dev_t rdev, dict_t *params)
+		 mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -720,15 +704,15 @@ error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_mknod_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev, params);
-	return 0;
+		    loc, mode, rdev, umask, xdata);
+        return 0;
 }
 
 
@@ -736,17 +720,17 @@ int
 error_gen_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno,
                              inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 int
 error_gen_mkdir (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, mode_t mode, dict_t *params)
+		 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -761,31 +745,32 @@ error_gen_mkdir (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_mkdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode, params);
-	return 0;
+		    loc, mode, umask, xdata);
+        return 0;
 }
 
 
 int
 error_gen_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                  dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -799,31 +784,34 @@ error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL,
+                                     xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_unlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
-	return 0;
+		    loc, xflag, xdata);
+        return 0;
 }
 
 
 int
 error_gen_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                 dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -837,15 +825,15 @@ error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_rmdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->rmdir,
-		    loc, flags);
-	return 0;
+		    loc, flags, xdata);
+        return 0;
 }
 
 
@@ -853,17 +841,17 @@ int
 error_gen_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		       int32_t op_ret, int32_t op_errno, inode_t *inode,
                        struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent)
+                       struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-		   loc_t *loc, dict_t *params)
+		   loc_t *loc, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -878,15 +866,15 @@ error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL); /* pre & post parent attr */
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_symlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc, params);
-	return 0;
+		    linkpath, loc, umask, xdata);
+        return 0;
 }
 
 
@@ -894,18 +882,19 @@ int
 error_gen_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                       struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent)
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
                              preoldparent, postoldparent,
-                             prenewparent, postnewparent);
-	return 0;
+                             prenewparent, postnewparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_rename (call_frame_t *frame, xlator_t *this,
-		  loc_t *oldloc, loc_t *newloc)
+		  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -920,15 +909,15 @@ error_gen_rename (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_rename_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc);
-	return 0;
+		    oldloc, newloc, xdata);
+        return 0;
 }
 
 
@@ -936,17 +925,17 @@ int
 error_gen_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		    int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_link (call_frame_t *frame, xlator_t *this,
-		loc_t *oldloc, loc_t *newloc)
+		loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -961,15 +950,15 @@ error_gen_link (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_link_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc);
-	return 0;
+		    oldloc, newloc, xdata);
+        return 0;
 }
 
 
@@ -977,17 +966,18 @@ int
 error_gen_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
 		      fd_t *fd, inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+		  int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                  dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -1002,30 +992,30 @@ error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL); /* pre & post attr */
+                                     NULL, NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_create_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd, params);
-	return 0;
+		    loc, flags, mode, umask, fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, fd_t *fd)
+		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
-	return 0;
+	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		int32_t flags, fd_t *fd, int32_t wbflags)
+		int32_t flags, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1039,15 +1029,15 @@ error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_open_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd, wbflags);
-	return 0;
+		    loc, flags, fd, xdata);
+        return 0;
 }
 
 
@@ -1055,17 +1045,17 @@ int
 error_gen_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno,
 		     struct iovec *vector, int32_t count,
-		     struct iatt *stbuf, struct iobref *iobref)
+		     struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref);
-	return 0;
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readv (call_frame_t *frame, xlator_t *this,
-		 fd_t *fd, size_t size, off_t offset, uint32_t flags)
+		 fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1080,33 +1070,33 @@ error_gen_readv (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 
 	STACK_WIND (frame, error_gen_readv_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readv,
-		    fd, size, offset, flags);
-	return 0;
+		    fd, size, offset, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *prebuf, struct iatt *postbuf)
+                      struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 		  struct iovec *vector, int32_t count,
-		  off_t off, uint32_t flags, struct iobref *iobref)
+		  off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1118,31 +1108,47 @@ error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (enable)
                 op_errno = error_gen (this, GF_FOP_WRITE);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+	if (op_errno == GF_ERROR_SHORT_WRITE) {
+		struct iovec *shortvec;
+
+		/*
+		 * A short write error returns some value less than what was
+		 * requested from a write. To simulate this, replace the vector
+		 * with one half the size;
+		 */
+		shortvec = iov_dup(vector, 1);
+		shortvec->iov_len /= 2;
+
+		STACK_WIND(frame, error_gen_writev_cbk, FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev, fd, shortvec, count,
+			   off, flags, iobref, xdata);
+		GF_FREE(shortvec);
 		return 0;
+	} else if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, xdata);
+        	return 0;
 	}
 
 	STACK_WIND (frame, error_gen_writev_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->writev,
-		    fd, vector, count, off, flags, iobref);
-	return 0;
+		    fd, vector, count, off, flags, iobref, xdata);
+        return 0;
 }
 
 
 int
 error_gen_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno)
+		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
-error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1156,15 +1162,15 @@ error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (flush, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_flush_cbk,
                     FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->flush,
-		    fd);
-	return 0;
+		    fd, xdata);
+        return 0;
 }
 
 
@@ -1172,15 +1178,15 @@ int
 error_gen_fsync_cbk (call_frame_t *frame, void *cookie,
 		     xlator_t *this, int32_t op_ret,
 		     int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
+	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1194,29 +1200,29 @@ error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fsync_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsync,
-		    fd, flags);
-	return 0;
+		    fd, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
-	return 0;
+	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1230,29 +1236,29 @@ error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fstat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fstat,
-		    fd);
-	return 0;
+		    fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, fd_t *fd)
+		       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
-	return 0;
+	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
 }
 
 
 int
-error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1266,29 +1272,29 @@ error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_opendir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd);
-	return 0;
+		    loc, fd, xdata);
+        return 0;
 }
 
 int
 error_gen_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+			int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    int32_t flags)
+                    int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1302,30 +1308,29 @@ error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fsyncdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsyncdir,
-		    fd, flags);
-	return 0;
+		    fd, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+		      int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
-
-	return 0;
+	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1339,31 +1344,30 @@ error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_statfs_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->statfs,
-		    loc);
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+			int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *dict, int32_t flags)
+		    dict_t *dict, int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1377,30 +1381,30 @@ error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->setxattr,
-		    loc, dict, flags);
-	return 0;
+		    loc, dict, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict)
+			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
-	return 0;
+	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    const char *name)
+		    const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1414,30 +1418,29 @@ error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_getxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, name);
-	return 0;
+		    loc, name, xdata);
+        return 0;
 }
 
 int
 error_gen_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     dict_t *dict, int32_t flags)
+                     dict_t *dict, int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1451,30 +1454,30 @@ error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fsetxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsetxattr,
-		    fd, dict, flags);
-	return 0;
+		    fd, dict, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *dict)
+                         int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
-	return 0;
+	STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name)
+                     const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1488,31 +1491,30 @@ error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fgetxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fgetxattr,
-		    fd, name);
-	return 0;
+		    fd, name, xdata);
+        return 0;
 }
 
 
 int
 error_gen_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
-
-	return 0;
+	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   gf_xattrop_flags_t flags, dict_t *dict)
+		   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1526,31 +1528,30 @@ error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_xattrop_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->xattrop,
-		    loc, flags, dict);
-	return 0;
+		    loc, flags, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict)
+			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
-
-	return 0;
+	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    gf_xattrop_flags_t flags, dict_t *dict)
+		    gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1564,31 +1565,30 @@ error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fxattrop_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fxattrop,
-		    fd, flags, dict);
-	return 0;
+		    fd, flags, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
+			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       const char *name)
+		       const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1602,30 +1602,29 @@ error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_removexattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->removexattr,
-		    loc, name);
-	return 0;
+		    loc, name, xdata);
+        return 0;
 }
 
 int
 error_gen_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
+			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        const char *name)
+                        const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1639,30 +1638,30 @@ error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fremovexattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fremovexattr,
-		    fd, name);
-	return 0;
+		    fd, name, xdata);
+        return 0;
 }
 
 
 int
 error_gen_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
-	return 0;
+	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
+        return 0;
 }
 
 
 int
 error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	      struct gf_flock *lock)
+	      struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1676,32 +1675,31 @@ error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_lk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->lk,
-		    fd, cmd, lock);
-	return 0;
+		    fd, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_inodelk_cbk (call_frame_t *frame, void *cookie,
-		       xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_inodelk (call_frame_t *frame, xlator_t *this,
 		   const char *volume, loc_t *loc, int32_t cmd,
-                   struct gf_flock *lock)
+                   struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1715,32 +1713,31 @@ error_gen_inodelk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_inodelk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->inodelk,
-		    volume, loc, cmd, lock);
-	return 0;
+		    volume, loc, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_finodelk_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_finodelk (call_frame_t *frame, xlator_t *this,
 		    const char *volume, fd_t *fd, int32_t cmd,
-                    struct gf_flock *lock)
+                    struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1754,32 +1751,31 @@ error_gen_finodelk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_finodelk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->finodelk,
-		    volume, fd, cmd, lock);
-	return 0;
+		    volume, fd, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_entrylk_cbk (call_frame_t *frame, void *cookie,
-		       xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_entrylk (call_frame_t *frame, xlator_t *this,
 		   const char *volume, loc_t *loc, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type)
+		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1793,32 +1789,31 @@ error_gen_entrylk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_entrylk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-	return 0;
+		    volume, loc, basename, cmd, type, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
 		    const char *volume, fd_t *fd, const char *basename,
-		    entrylk_cmd cmd, entrylk_type type)
+		    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -1832,15 +1827,15 @@ error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno, xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fentrylk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fentrylk,
-		    volume, fd, basename, cmd, type);
-	return 0;
+		    volume, fd, basename, cmd, type, xdata);
+        return 0;
 }
 
 
@@ -1852,8 +1847,7 @@ error_gen_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		       int32_t op_ret, int32_t op_errno, char *spec_data)
 {
 	STACK_UNWIND_STRICT (getspec, frame, op_ret, op_errno, spec_data);
-
-	return 0;
+        return 0;
 }
 
 
@@ -1887,16 +1881,17 @@ error_gen_getspec (call_frame_t *frame, xlator_t *this, const char *key,
 
 int
 error_gen_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
+	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
 
 int
 error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		   size_t size, off_t off)
+		   size_t size, off_t off, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1910,23 +1905,24 @@ error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL, xdata);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_readdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readdir,
-		    fd, size, off);
+		    fd, size, off, xdata);
 	return 0;
 }
 
 
 int
 error_gen_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
@@ -1947,7 +1943,7 @@ error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
 		return 0;
 	}
 
@@ -1958,18 +1954,83 @@ error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 	return 0;
 }
 
-
-int
-error_gen_closedir (xlator_t *this, fd_t *fd)
+static void
+error_gen_set_failure (eg_t *pvt, int percent)
 {
-	return 0;
+        GF_ASSERT (pvt);
+
+        if (percent)
+                pvt->failure_iter_no = 100/percent;
+        else
+                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
 }
 
+static void
+error_gen_parse_fill_fops (eg_t *pvt, char *enable_fops)
+{
+        char            *op_no_str = NULL;
+        int              op_no = -1;
+        int              i = 0;
+        xlator_t        *this = THIS;
+        char            *saveptr = NULL;
 
-int
-error_gen_close (xlator_t *this, fd_t *fd)
+        GF_ASSERT (pvt);
+        GF_ASSERT (this);
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                pvt->enable[i] = 0;
+
+        if (!enable_fops) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "All fops are enabled.");
+                for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                        pvt->enable[i] = 1;
+        } else {
+                op_no_str = strtok_r (enable_fops, ",", &saveptr);
+                while (op_no_str) {
+                        op_no = get_fop_int (&op_no_str);
+                        if (op_no == -1) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Wrong option value %s", op_no_str);
+                        } else
+                                pvt->enable[op_no] = 1;
+
+                        op_no_str = strtok_r (NULL, ",", &saveptr);
+                }
+        }
+}
+
+int32_t
+error_gen_priv_dump (xlator_t *this)
 {
-	return 0;
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        int             ret = -1;
+        eg_t            *conf = NULL;
+
+        if (!this)
+            goto out;
+
+        conf = this->private;
+        if (!conf)
+            goto out;
+
+        ret = TRY_LOCK(&conf->lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.debug.error-gen","%s.priv",
+                               this->name);
+
+        gf_proc_dump_write("op_count", "%d", conf->op_count);
+        gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
+        gf_proc_dump_write("error_no", "%s", conf->error_no);
+        gf_proc_dump_write("random_failure", "%d", conf->random_failure);
+
+        UNLOCK(&conf->lock);
+out:
+        return ret;
 }
 
 int32_t
@@ -1992,18 +2053,43 @@ mem_acct_init (xlator_t *this)
 }
 
 int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        eg_t            *pvt = NULL;
+        int32_t          ret = 0;
+        char            *error_enable_fops = NULL;
+        int32_t          failure_percent_int = 0;
+
+        if (!this || !this->private)
+                goto out;
+
+        pvt = this->private;
+
+        GF_OPTION_RECONF ("error-no", pvt->error_no, options, str, out);
+
+        GF_OPTION_RECONF ("failure", failure_percent_int, options, int32,
+                          out);
+
+        GF_OPTION_RECONF ("enable", error_enable_fops, options, str, out);
+
+        GF_OPTION_RECONF ("random-failure", pvt->random_failure, options,
+                          bool, out);
+
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
+        return ret;
+}
+
+int
 init (xlator_t *this)
 {
         eg_t            *pvt = NULL;
-        data_t          *error_no = NULL;
-        data_t          *failure_percent = NULL;
-        data_t          *enable = NULL;
-        gf_boolean_t    random_failure = _gf_false;
         int32_t          ret = 0;
         char            *error_enable_fops = NULL;
-        char            *op_no_str = NULL;
-        int              op_no = -1;
-        int              i = 0;
         int32_t          failure_percent_int = 0;
 
         if (!this->children || this->children->next) {
@@ -2018,79 +2104,34 @@ init (xlator_t *this)
                         "dangling volume. check volfile ");
         }
 
-        error_no = dict_get (this->options, "error-no");
-        failure_percent = dict_get (this->options, "failure");
-        enable = dict_get (this->options, "enable");
-
         pvt = GF_CALLOC (1, sizeof (eg_t), gf_error_gen_mt_eg_t);
 
         if (!pvt) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "out of memory.");
                 ret = -1;
                 goto out;
         }
 
         LOCK_INIT (&pvt->lock);
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                pvt->enable[i] = 0;
-        if (!error_no) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "error-no not specified.");
-        } else {
-                pvt->error_no = data_to_str (error_no);
-        }
+        GF_OPTION_INIT ("error-no", pvt->error_no, str, out);
 
-        if (!failure_percent) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failure percent not specified.");
-                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
-        } else {
-                failure_percent_int = data_to_int32 (failure_percent);
-                if (failure_percent_int)
-                        pvt->failure_iter_no = 100/failure_percent_int;
-                else
-                        pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
-        }
+        GF_OPTION_INIT ("failure", failure_percent_int, int32, out);
+
+        GF_OPTION_INIT ("enable", error_enable_fops, str, out);
+
+        GF_OPTION_INIT ("random-failure", pvt->random_failure, bool, out);
 
-        if (!enable) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "All fops are enabled.");
-                for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                        pvt->enable[i] = 1;
-        } else {
-                error_enable_fops = data_to_str (enable);
-                op_no_str = error_enable_fops;
-                while ((*error_enable_fops) != '\0') {
-                        error_enable_fops++;
-                        if (((*error_enable_fops) == ',') ||
-                            ((*error_enable_fops) == '\0')) {
-                                if ((*error_enable_fops) != '\0') {
-                                        (*error_enable_fops) = '\0';
-                                        error_enable_fops++;
-                                }
-                                op_no = get_fop_int (&op_no_str);
-                                if (op_no == -1) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "Wrong option value %s",
-                                                op_no_str);
-                                } else
-                                        pvt->enable[op_no] = 1;
-                                op_no_str = error_enable_fops;
-                        }
-                }
-        }
 
-        random_failure = dict_get_str_boolean (this->options, "random-failure",
-                                               _gf_false);
-        pvt->random_failure = random_failure;
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
 
         this->private = pvt;
 
         /* Give some seed value here */
         srand (time(NULL));
 out:
+        if (ret)
+                GF_FREE (pvt);
         return ret;
 }
 
@@ -2112,6 +2153,12 @@ fini (xlator_t *this)
         return;
 }
 
+struct xlator_dumpops dumpops = {
+        .priv = error_gen_priv_dump,
+};
+
+struct xlator_fops cbks;
+
 struct xlator_fops fops = {
 	.lookup      = error_gen_lookup,
 	.stat        = error_gen_stat,
@@ -2157,24 +2204,29 @@ struct xlator_fops fops = {
 	.getspec     = error_gen_getspec,
 };
 
-struct xlator_cbks cbks = {
-	.release = error_gen_close,
-	.releasedir = error_gen_closedir,
-};
-
 struct volume_options options[] = {
         { .key  = {"failure"},
-          .type = GF_OPTION_TYPE_INT },
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Percentage failure of operations when enabled.",
+        },
+
         { .key  = {"error-no"},
           .value = {"ENOENT","ENOTDIR","ENAMETOOLONG","EACCES","EBADF",
                     "EFAULT","ENOMEM","EINVAL","EIO","EEXIST","ENOSPC",
                     "EPERM","EROFS","EBUSY","EISDIR","ENOTEMPTY","EMLINK"
                     "ENODEV","EXDEV","EMFILE","ENFILE","ENOSYS","EINTR",
-                    "EFBIG","EAGAIN"},
-          .type = GF_OPTION_TYPE_STR },
+                    "EFBIG","EAGAIN","GF_ERROR_SHORT_WRITE"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
         { .key = {"random-failure"},
-          .type = GF_OPTION_TYPE_BOOL},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+
         { .key  = {"enable"},
-          .type = GF_OPTION_TYPE_STR },
+          .type = GF_OPTION_TYPE_STR,
+        },
+
         { .key  = {NULL} }
 };
diff --git a/xlators/debug/error-gen/src/error-gen.h b/xlators/debug/error-gen/src/error-gen.h
index bb3adb2ab..d92c23062 100644
--- a/xlators/debug/error-gen/src/error-gen.h
+++ b/xlators/debug/error-gen/src/error-gen.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _ERROR_GEN_H
 #define _ERROR_GEN_H
 
@@ -29,6 +19,18 @@
 
 #define GF_FAILURE_DEFAULT 10
 
+/*
+ * Pseudo-errors refer to errors beyond the scope of traditional <-1, op_errno>
+ * returns. This facilitates the ability to return unexpected, but not -1 values
+ * and/or to inject operations that lead to implicit error conditions. The range
+ * for pseudo errors resides at a high value to avoid conflicts with the errno
+ * range.
+ */
+enum GF_PSEUDO_ERRORS {
+	GF_ERROR_SHORT_WRITE = 1000,	/* short writev return value */
+	GF_ERROR_MAX
+};
+
 typedef struct {
         int enable[GF_FOP_MAXVALUE];
         int op_count;
diff --git a/xlators/debug/io-stats/src/Makefile.am b/xlators/debug/io-stats/src/Makefile.am
index b894e79c3..dff294cd8 100644
--- a/xlators/debug/io-stats/src/Makefile.am
+++ b/xlators/debug/io-stats/src/Makefile.am
@@ -2,14 +2,17 @@
 xlator_LTLIBRARIES = io-stats.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-io_stats_la_LDFLAGS = -module -avoidversion
+io_stats_la_LDFLAGS = -module -avoid-version
 
 io_stats_la_SOURCES = io-stats.c
 io_stats_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = io-stats-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index 2063f6d6a..c30dfb17e 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __IO_STATS_MEM_TYPES_H__
 #define __IO_STATS_MEM_TYPES_H__
 
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 5f1e2bb78..9e48a7c6e 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -46,6 +36,7 @@
 #include <stdarg.h>
 #include "defaults.h"
 #include "logging.h"
+#include "cli1-xdr.h"
 
 #define MAX_LIST_MEMBERS 100
 
@@ -58,7 +49,7 @@ typedef enum {
         IOS_STATS_TYPE_READDIRP,
         IOS_STATS_TYPE_READ_THROUGHPUT,
         IOS_STATS_TYPE_WRITE_THROUGHPUT,
-        IOS_STATS_TYPE_MAX,
+        IOS_STATS_TYPE_MAX
 }ios_stats_type_t;
 
 typedef enum {
@@ -293,43 +284,43 @@ is_fop_latency_started (call_frame_t *frame)
                 }                                                               \
                 UNLOCK (&iosstat->lock);                                        \
                 ios_stat_add_to_list (&conf->list[type],                        \
-				     value, iosstat);                           \
+                                     value, iosstat);                           \
                                                                                 \
         } while (0)
 
 
 #define BUMP_THROUGHPUT(iosstat, type)						\
         do {									\
-	        struct ios_conf         *conf = NULL;				\
-		double                   elapsed;				\
-		struct timeval          *begin, *end;				\
-		double                   throughput;				\
+                struct ios_conf         *conf = NULL;				\
+                double                   elapsed;				\
+                struct timeval          *begin, *end;				\
+                double                   throughput;				\
                int                      flag = 0;                              \
-										\
-		begin = &frame->begin;						\
-		end   = &frame->end;						\
-										\
-		elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
-			+ (end->tv_usec - begin->tv_usec);			\
-		throughput = op_ret / elapsed;					\
-										\
-		conf = this->private;						\
-		LOCK(&iosstat->lock);						\
-		{								\
-			if (iosstat->thru_counters[type].throughput             \
+                                                                                \
+                begin = &frame->begin;						\
+                end   = &frame->end;						\
+                                                                                \
+                elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
+                        + (end->tv_usec - begin->tv_usec);			\
+                throughput = op_ret / elapsed;					\
+                                                                                \
+                conf = this->private;						\
+                LOCK(&iosstat->lock);						\
+                {								\
+                        if (iosstat->thru_counters[type].throughput             \
                                 <= throughput) {                                \
-				iosstat->thru_counters[type].throughput =       \
+                                iosstat->thru_counters[type].throughput =       \
                                                                 throughput;     \
-				gettimeofday (&iosstat->                        \
+                                gettimeofday (&iosstat->                        \
                                              thru_counters[type].time, NULL);   \
                                flag = 1;                                       \
                         }							\
                 }								\
-        	UNLOCK (&iosstat->lock);					\
+                UNLOCK (&iosstat->lock);					\
                if (flag)                                                       \
                        ios_stat_add_to_list (&conf->thru_list[type],           \
                                                throughput, iosstat);           \
-	} while (0)
+        } while (0)
 
 int
 ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
@@ -484,12 +475,12 @@ ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
                         new = GF_CALLOC (1, sizeof (*new),
                                         gf_io_stats_mt_ios_stat_list);
                         new->iosstat = iosstat;
-                        new->value = value; 
+                        new->value = value;
                         ios_stat_ref (iosstat);
-                        list_add_tail (&new->list, &tmp->list);  
+                        list_add_tail (&new->list, &tmp->list);
                         stat = last->iosstat;
                         last->iosstat = NULL;
-                        ios_stat_unref (stat); 
+                        ios_stat_unref (stat);
                         list_del (&last->list);
                         GF_FREE (last);
                         if (reposition == MAX_LIST_MEMBERS)
@@ -511,7 +502,7 @@ ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
                         list_head->members++;
                         if (list_head->min_cnt > value)
                                 list_head->min_cnt = value;
-                } 
+                }
         }
 out:
         UNLOCK (&list_head->lock);
@@ -568,19 +559,16 @@ ios_dump_throughput_stats (struct ios_stat_head *list_head, xlator_t *this,
                             FILE* logfp, ios_stats_type_t type)
 {
         struct ios_stat_list *entry = NULL;
-        struct timeval        time = {0, };
-        struct tm             *tm = NULL;
+        struct timeval        time  = {0, };
         char                  timestr[256] = {0, };
 
         LOCK (&list_head->lock);
         {
                 list_for_each_entry (entry, &list_head->iosstats->list, list) {
-                        time = entry->iosstat->thru_counters[type].time;
-                        tm    = localtime (&time.tv_sec);
-                        if (!tm)
-                                continue;
-                        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                        gf_time_fmt (timestr, sizeof timestr,
+                                     entry->iosstat->thru_counters[type].time.tv_sec,
+                                     gf_timefmt_FT);
+                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
                           ".%"GF_PRI_SUSECONDS, time.tv_usec);
 
                         ios_log (this, logfp, "%s \t %-10.2f  \t  %s",
@@ -600,7 +588,6 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
         int                   index = 0;
         struct ios_stat_head *list_head = NULL;
         struct ios_conf      *conf = NULL;
-        struct tm            *tm = NULL;
         char                  timestr[256] = {0, };
         char                  str_header[128] = {0};
         char                  str_read[128] = {0};
@@ -694,9 +681,10 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
         if (interval == -1) {
                 LOCK (&conf->lock);
                 {
-                        tm = localtime (&conf->cumulative.max_openfd_time.tv_sec);
-                        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                        gf_time_fmt (timestr, sizeof timestr,
+                                     conf->cumulative.max_openfd_time.tv_sec,
+                                     gf_timefmt_FT);
+                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
                                   ".%"GF_PRI_SUSECONDS,
                                   conf->cumulative.max_openfd_time.tv_usec);
                         ios_log (this, logfp, "Current open fd's: %"PRId64
@@ -735,13 +723,13 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
                 ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
                          "\tFILE NAME");
                 list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-		ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
+                ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
 
                 ios_log (this, logfp, "\n======Write Throughput File Stats======");
                 ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
                          "\tFILE NAME");
                 list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-		ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_THRU_WRITE);
+                ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_THRU_WRITE);
         }
         return 0;
 }
@@ -929,8 +917,19 @@ ios_dump_args_init (struct ios_dump_args *args, ios_dump_type_t type,
         return ret;
 }
 
+static void
+ios_global_stats_clear (struct ios_global_stats *stats, struct timeval *now)
+{
+        GF_ASSERT (stats);
+        GF_ASSERT (now);
+
+        memset (stats, 0, sizeof (*stats));
+        stats->started_at = *now;
+}
+
 int
-io_stats_dump (xlator_t *this, struct ios_dump_args *args)
+io_stats_dump (xlator_t *this, struct ios_dump_args *args,
+               gf1_cli_info_op op, gf_boolean_t is_peek)
 {
         struct ios_conf         *conf = NULL;
         struct ios_global_stats  cumulative = {0, };
@@ -948,18 +947,32 @@ io_stats_dump (xlator_t *this, struct ios_dump_args *args)
         gettimeofday (&now, NULL);
         LOCK (&conf->lock);
         {
-                cumulative  = conf->cumulative;
-                incremental = conf->incremental;
+                if (op == GF_CLI_INFO_ALL ||
+                    op == GF_CLI_INFO_CUMULATIVE)
+                        cumulative  = conf->cumulative;
+
+                if (op == GF_CLI_INFO_ALL ||
+                    op == GF_CLI_INFO_INCREMENTAL) {
+                        incremental = conf->incremental;
+                        increment = conf->increment;
 
-                increment = conf->increment++;
+                        if (!is_peek) {
+                                increment = conf->increment++;
 
-                memset (&conf->incremental, 0, sizeof (conf->incremental));
-                conf->incremental.started_at = now;
+                                ios_global_stats_clear (&conf->incremental,
+                                                        &now);
+                        }
+                }
         }
         UNLOCK (&conf->lock);
 
-        io_stats_dump_global (this, &cumulative, &now, -1, args);
-        io_stats_dump_global (this, &incremental, &now, increment, args);
+        if (op == GF_CLI_INFO_ALL ||
+            op == GF_CLI_INFO_CUMULATIVE)
+                io_stats_dump_global (this, &cumulative, &now, -1, args);
+
+        if (op == GF_CLI_INFO_ALL ||
+            op == GF_CLI_INFO_INCREMENTAL)
+                io_stats_dump_global (this, &incremental, &now, increment, args);
 
         return 0;
 }
@@ -1080,36 +1093,45 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
         struct ios_stat_list    *entry = NULL;
         int                      ret = -1;
         ios_stats_thru_t         index = IOS_STATS_THRU_MAX;
-        struct tm               *tm = NULL;
         char                     timestr[256] = {0, };
+        char                    *dict_timestr = NULL;
 
         conf = this->private;
 
         switch (flags) {
-                case IOS_STATS_TYPE_OPEN: 
+                case IOS_STATS_TYPE_OPEN:
                         list_head = &conf->list[IOS_STATS_TYPE_OPEN];
                         LOCK (&conf->lock);
                         {
                                 ret = dict_set_uint64 (resp, "current-open",
                                                      conf->cumulative.nr_opens);
                                 if (ret)
-                                        goto out;
+                                        goto unlock;
                                 ret = dict_set_uint64 (resp, "max-open",
                                                        conf->cumulative.max_nr_opens);
 
-                                tm = localtime (&conf->cumulative.max_openfd_time.tv_sec);
-                                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-                                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                                          ".%"GF_PRI_SUSECONDS,
-                                          conf->cumulative.max_openfd_time.tv_usec);
-
-                                ret = dict_set_str (resp, "max-openfd-time",
-                                                    timestr);
+                                gf_time_fmt (timestr, sizeof timestr,
+                                             conf->cumulative.max_openfd_time.tv_sec,
+                                             gf_timefmt_FT);
+                                if (conf->cumulative.max_openfd_time.tv_sec)
+                                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                                                  ".%"GF_PRI_SUSECONDS,
+                                                  conf->cumulative.max_openfd_time.tv_usec);
+
+                                dict_timestr = gf_strdup (timestr);
+                                if (!dict_timestr)
+                                        goto unlock;
+                                ret = dict_set_dynstr (resp, "max-openfd-time",
+                                                       dict_timestr);
                                 if (ret)
-                                        goto out;
+                                        goto unlock;
                         }
+        unlock:
                         UNLOCK (&conf->lock);
-
+                        /* Do not proceed if we came here because of some error
+                         * during the dict operation */
+                        if (ret)
+                                goto out;
                         break;
                 case IOS_STATS_TYPE_READ:
                         list_head = &conf->list[IOS_STATS_TYPE_READ];
@@ -1125,7 +1147,7 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
                         break;
                 case IOS_STATS_TYPE_READ_THROUGHPUT:
                         list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                        index = IOS_STATS_THRU_READ; 
+                        index = IOS_STATS_THRU_READ;
                         break;
                 case IOS_STATS_TYPE_WRITE_THROUGHPUT:
                         list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
@@ -1146,39 +1168,44 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
                         snprintf (key, 256, "%s-%d", "filename", cnt);
                         ret = dict_set_str (resp, key, entry->iosstat->filename);
                         if (ret)
-                                goto out;
+                                goto unlock_list_head;
                          snprintf (key, 256, "%s-%d", "value",cnt);
                          ret = dict_set_uint64 (resp, key, entry->value);
                          if (ret)
-                                 goto out;
+                                 goto unlock_list_head;
                          if (index != IOS_STATS_THRU_MAX) {
                                  snprintf (key, 256, "%s-%d", "time-sec", cnt);
-                                 ret = dict_set_int32 (resp, key, 
+                                 ret = dict_set_int32 (resp, key,
                                          entry->iosstat->thru_counters[index].time.tv_sec);
                                  if (ret)
-                                         goto out;
+                                         goto unlock_list_head;
                                  snprintf (key, 256, "%s-%d", "time-usec", cnt);
-                                 ret = dict_set_int32 (resp, key, 
+                                 ret = dict_set_int32 (resp, key,
                                          entry->iosstat->thru_counters[index].time.tv_usec);
                                  if (ret)
-                                         goto out;
+                                         goto unlock_list_head;
                          }
                          if (cnt == list_cnt)
                                  break;
 
                 }
         }
+unlock_list_head:
         UNLOCK (&list_head->lock);
-
+        /* ret is !=0 if some dict operation in the above critical region
+         * failed. */
+        if (ret)
+                goto out;
         ret = dict_set_int32 (resp, "members", cnt);
  out:
         return ret;
 }
+
 int
 io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, fd_t *fd,
                      inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         struct ios_fd *iosfd = NULL;
         char          *path = NULL;
@@ -1231,21 +1258,21 @@ io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unwind:
         UPDATE_PROFILE_STATS (frame, CREATE);
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd)
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct ios_fd *iosfd = NULL;
         char          *path = NULL;
         struct   ios_stat *iosstat = NULL;
         struct ios_conf   *conf = NULL;
 
-	conf = this->private;
+        conf = this->private;
         path = frame->local;
         frame->local = NULL;
 
@@ -1269,6 +1296,16 @@ io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         ios_fd_ctx_set (fd, this, iosfd);
 
         ios_inode_ctx_get (fd->inode, this, &iosstat);
+        if (!iosstat) {
+                iosstat = GF_CALLOC (1, sizeof (*iosstat),
+                                     gf_io_stats_mt_ios_stat);
+                if (iosstat) {
+                        iosstat->filename = gf_strdup (path);
+                        uuid_copy (iosstat->gfid, fd->inode->gfid);
+                        LOCK_INIT (&iosstat->lock);
+                        ios_inode_ctx_set (fd->inode, this, iosstat);
+                }
+        }
 
         LOCK (&conf->lock);
         {
@@ -1286,7 +1323,7 @@ io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unwind:
         UPDATE_PROFILE_STATS (frame, OPEN);
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 
 }
@@ -1294,10 +1331,10 @@ unwind:
 
 int
 io_stats_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, STAT);
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
@@ -1306,7 +1343,7 @@ int
 io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     struct iovec *vector, int32_t count,
-                    struct iatt *buf, struct iobref *iobref)
+                    struct iatt *buf, struct iobref *iobref, dict_t *xdata)
 {
         int              len = 0;
         fd_t            *fd = NULL;
@@ -1330,7 +1367,7 @@ io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, buf, iobref);
+                             vector, count, buf, iobref, xdata);
         return 0;
 
 }
@@ -1339,7 +1376,7 @@ io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf)
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         struct ios_stat *iosstat = NULL;
         inode_t         *inode   = NULL;
@@ -1351,13 +1388,13 @@ io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 ios_inode_ctx_get (inode, this, &iosstat);
                 if (iosstat) {
                         BUMP_STATS (iosstat, IOS_STATS_TYPE_WRITE);
-			BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
+                        BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
                         inode = NULL;
                         iosstat = NULL;
                 }
         }
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
         return 0;
 
 }
@@ -1367,7 +1404,7 @@ io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
 {
         struct ios_stat *iosstat = NULL;
         inode_t         *inode   = frame->local;
@@ -1383,17 +1420,17 @@ io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               iosstat = NULL;
         }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
 io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, READDIR);
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
@@ -1401,10 +1438,10 @@ io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf)
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FSYNC);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
         return 0;
 }
 
@@ -1412,10 +1449,10 @@ io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, SETATTR);
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
         return 0;
 }
 
@@ -1423,11 +1460,11 @@ io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, UNLINK);
         STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 
 }
@@ -1437,12 +1474,12 @@ int
 io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                      struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent)
+                     struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, RENAME);
         STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
                              preoldparent, postoldparent,
-                             prenewparent, postnewparent);
+                             prenewparent, postnewparent, xdata);
         return 0;
 }
 
@@ -1450,10 +1487,10 @@ io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, const char *buf,
-                       struct iatt *sbuf)
+                       struct iatt *sbuf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, READLINK);
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
         return 0;
 }
 
@@ -1462,10 +1499,10 @@ int
 io_stats_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
                      inode_t *inode, struct iatt *buf,
-                     dict_t *xattr, struct iatt *postparent)
+                     dict_t *xdata, struct iatt *postparent)
 {
         UPDATE_PROFILE_STATS (frame, LOOKUP);
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xattr,
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
                              postparent);
         return 0;
 }
@@ -1475,11 +1512,11 @@ int
 io_stats_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, SYMLINK);
         STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -1488,11 +1525,11 @@ int
 io_stats_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, MKNOD);
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -1501,7 +1538,8 @@ int
 io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata)
 {
         struct ios_stat *iosstat = NULL;
         char   *path = frame->local;
@@ -1520,11 +1558,10 @@ io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 unwind:
         /* local is assigned with path */
-        if (frame->local)
-                GF_FREE (frame->local);
+        GF_FREE (frame->local);
         frame->local = NULL;
         STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -1533,28 +1570,28 @@ int
 io_stats_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent)
+                   struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, LINK);
         STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 io_stats_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FLUSH);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd)
+                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct ios_stat *iosstat = NULL;
         int              ret     = -1;
@@ -1570,7 +1607,7 @@ io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 BUMP_STATS (iosstat, IOS_STATS_TYPE_OPENDIR);
 
 unwind:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
@@ -1578,13 +1615,13 @@ unwind:
 int
 io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
 
         UPDATE_PROFILE_STATS (frame, RMDIR);
 
         STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -1592,100 +1629,100 @@ io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *prebuf, struct iatt *postbuf)
+                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, TRUNCATE);
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 int
 io_stats_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                     int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, STATFS);
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
 io_stats_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, SETXATTR);
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, GETXATTR);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, REMOVEXATTR);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 io_stats_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FSETXATTR);
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *dict)
+                        int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FGETXATTR);
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FREMOVEXATTR);
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FSYNCDIR);
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, ACCESS);
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -1693,92 +1730,126 @@ io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno,
-                        struct iatt *prebuf, struct iatt *postbuf)
+                        struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FTRUNCATE);
         STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 int
 io_stats_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FSTAT);
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
+io_stats_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		       struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, FALLOCATE);
+	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+
+int
+io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		     struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, DISCARD);
+	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+int
+io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS(frame, ZEROFILL);
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
+
+int
 io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, LK);
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
 
 int
 io_stats_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, ENTRYLK);
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, XATTROP);
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, FXATTROP);
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         UPDATE_PROFILE_STATS (frame, INODELK);
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 io_stats_entrylk (call_frame_t *frame, xlator_t *this,
                   const char *volume, loc_t *loc, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type)
+                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_entrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
 
 int
 io_stats_inodelk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
 
         START_FOP_LATENCY (frame);
@@ -1786,128 +1857,122 @@ io_stats_inodelk (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_inodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock);
+                    volume, loc, cmd, flock, xdata);
         return 0;
 }
 
 
 int
 io_stats_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
 
         UPDATE_PROFILE_STATS (frame, FINODELK);
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
-io_stats_finodelk (call_frame_t *frame, xlator_t *this,
-                   const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+io_stats_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_finodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock);
+                    volume, fd, cmd, flock, xdata);
         return 0;
 }
 
 
 int
-io_stats_xattrop (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, gf_xattrop_flags_t flags, dict_t *dict)
+io_stats_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_xattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict);
-
+                    loc, flags, dict, xdata);
         return 0;
 }
 
 
 int
-io_stats_fxattrop (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+io_stats_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fxattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict);
-
+                    fd, flags, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_lookup (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xattr_req)
+                 loc_t *loc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
-
+                    loc, xdata);
         return 0;
 }
 
 
 int
-io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_stat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->stat,
-                    loc);
-
+                    loc, xdata);
         return 0;
 }
 
 
 int
 io_stats_readlink (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, size_t size)
+                   loc_t *loc, size_t size, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
-                    loc, size);
-
+                    loc, size, xdata);
         return 0;
 }
 
 
 int
-io_stats_mknod (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, dev_t dev, dict_t *params)
+io_stats_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, params);
-
+                    loc, mode, dev, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_mkdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, dict_t *params)
+                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
         frame->local = gf_strdup (loc->path);
 
@@ -1916,117 +1981,112 @@ io_stats_mkdir (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_mkdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_unlink (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc)
+                 loc_t *loc, int xflag, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_unlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->unlink,
-                    loc);
+                    loc, xflag, xdata);
         return 0;
 }
 
 
 int
 io_stats_rmdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags)
+                loc_t *loc, int flags, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_rmdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags);
-
+                    loc, flags, xdata);
         return 0;
 }
 
 
 int
-io_stats_symlink (call_frame_t *frame, xlator_t *this,
-                  const char *linkpath, loc_t *loc, dict_t *params)
+io_stats_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, params);
-
+                    linkpath, loc, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_rename (call_frame_t *frame, xlator_t *this,
-                 loc_t *oldloc, loc_t *newloc)
+                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_rename_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc);
-
+                    oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
 io_stats_link (call_frame_t *frame, xlator_t *this,
-               loc_t *oldloc, loc_t *newloc)
+               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_link_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
 io_stats_setattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, struct iatt *stbuf, int32_t valid)
+                  loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid);
-
+                    loc, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
 io_stats_truncate (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, off_t offset)
+                   loc_t *loc, off_t offset, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_truncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->truncate,
-                    loc, offset);
-
+                    loc, offset, xdata);
         return 0;
 }
 
 
 int
-io_stats_open (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, int32_t flags, fd_t *fd, int32_t wbflags)
+io_stats_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int32_t flags, fd_t *fd, dict_t *xdata)
 {
         frame->local = gf_strdup (loc->path);
 
@@ -2035,7 +2095,7 @@ io_stats_open (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_open_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
         return 0;
 }
 
@@ -2043,7 +2103,7 @@ io_stats_open (call_frame_t *frame, xlator_t *this,
 int
 io_stats_create (call_frame_t *frame, xlator_t *this,
                  loc_t *loc, int32_t flags, mode_t mode,
-                 fd_t *fd, dict_t *params)
+                 mode_t umask, fd_t *fd, dict_t *xdata)
 {
         frame->local = gf_strdup (loc->path);
 
@@ -2052,14 +2112,14 @@ io_stats_create (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
 
 int
 io_stats_readv (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset, uint32_t flags)
+                fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         frame->local = fd;
 
@@ -2068,7 +2128,7 @@ io_stats_readv (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_readv_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
@@ -2077,7 +2137,7 @@ int
 io_stats_writev (call_frame_t *frame, xlator_t *this,
                  fd_t *fd, struct iovec *vector,
                  int32_t count, off_t offset,
-                 uint32_t flags, struct iobref *iobref)
+                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         int                 len = 0;
 
@@ -2091,7 +2151,7 @@ io_stats_writev (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
         return 0;
 
 }
@@ -2099,47 +2159,47 @@ io_stats_writev (call_frame_t *frame, xlator_t *this,
 
 int
 io_stats_statfs (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc)
+                 loc_t *loc, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_statfs_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->statfs,
-                    loc);
+                    loc, xdata);
         return 0;
 }
 
 
 int
 io_stats_flush (call_frame_t *frame, xlator_t *this,
-                fd_t *fd)
+                fd_t *fd, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_flush_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->flush,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsync (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t flags)
+                fd_t *fd, int32_t flags, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fsync_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsync,
-                    fd, flags);
+                    fd, flags, xdata);
         return 0;
 }
 
 
-void
+int
 conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 {
         struct {
@@ -2163,27 +2223,27 @@ conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 
                 if (!strncmp (filename, "", 1)) {
                         gf_log (this->name, GF_LOG_ERROR, "No filename given");
-                        return;
+                        return -1;
                 }
                 logfp = fopen (filename, "w+");
-                GF_ASSERT (logfp);
                 if (!logfp) {
                         gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
                                 "for writing", filename);
-                        return;
-                }
+                        return -1;
+                    }
                 (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
                                            logfp);
-                io_stats_dump (this, &args);
+                io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
                 fclose (logfp);
         }
+        return 0;
 }
 
 
 int
 io_stats_setxattr (call_frame_t *frame, xlator_t *this,
                    loc_t *loc, dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
         struct {
                 xlator_t     *this;
@@ -2202,36 +2262,35 @@ io_stats_setxattr (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 }
 
 
 int
 io_stats_getxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name)
+                   loc_t *loc, const char *name, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 }
 
 
 int
 io_stats_removexattr (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, const char *name)
+                      loc_t *loc, const char *name, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_removexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->removexattr,
-                    loc, name);
-
+                    loc, name, xdata);
         return 0;
 }
 
@@ -2239,50 +2298,49 @@ io_stats_removexattr (call_frame_t *frame, xlator_t *this,
 int
 io_stats_fsetxattr (call_frame_t *frame, xlator_t *this,
                     fd_t *fd, dict_t *dict,
-                    int32_t flags)
+                    int32_t flags, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fsetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, dict, flags, xdata);
         return 0;
 }
 
 
 int
 io_stats_fgetxattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, const char *name)
+                    fd_t *fd, const char *name, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fgetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name);
+                    fd, name, xdata);
         return 0;
 }
 
 
 int
 io_stats_fremovexattr (call_frame_t *frame, xlator_t *this,
-                       fd_t *fd, const char *name)
+                       fd_t *fd, const char *name, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fremovexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name);
-
+                    fd, name, xdata);
         return 0;
 }
 
 
 int
 io_stats_opendir (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, fd_t *fd)
+                  loc_t *loc, fd_t *fd, dict_t *xdata)
 {
 
         START_FOP_LATENCY (frame);
@@ -2290,7 +2348,7 @@ io_stats_opendir (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
-                    loc, fd);
+                    loc, fd, xdata);
         return 0;
 }
 
@@ -2305,107 +2363,143 @@ io_stats_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdirp,
                     fd, size, offset, dict);
-
         return 0;
 }
 
 
 int
 io_stats_readdir (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset)
+                  fd_t *fd, size_t size, off_t offset, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset);
-
+                    fd, size, offset, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsyncdir (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, int32_t datasync)
+                   fd_t *fd, int32_t datasync, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fsyncdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync);
+                    fd, datasync, xdata);
         return 0;
 }
 
 
 int
 io_stats_access (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t mask)
+                 loc_t *loc, int32_t mask, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_access_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
         return 0;
 }
 
 
 int
 io_stats_ftruncate (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, off_t offset)
+                    fd_t *fd, off_t offset, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_ftruncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset);
-
+                    fd, offset, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsetattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, struct iatt *stbuf, int32_t valid)
+                   fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid);
+                    fd, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
 io_stats_fstat (call_frame_t *frame, xlator_t *this,
-                fd_t *fd)
+                fd_t *fd, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fstat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fstat,
-                    fd);
+                    fd, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		   off_t offset, size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		   xdata);
+
+	return 0;
+}
+
+
+int
+io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+		 size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+
+	return 0;
+}
+
+int
+io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
+{
+        START_FOP_LATENCY(frame);
+
+        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+
         return 0;
 }
 
 
 int
 io_stats_lk (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock)
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_lk_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock);
+                    fd, cmd, lock, xdata);
         return 0;
 }
 
@@ -2422,7 +2516,7 @@ io_stats_release (xlator_t *this, fd_t *fd)
 
         LOCK (&conf->lock);
         {
-		conf->cumulative.nr_opens--;
+                conf->cumulative.nr_opens--;
         }
         UNLOCK (&conf->lock);
 
@@ -2430,8 +2524,7 @@ io_stats_release (xlator_t *this, fd_t *fd)
         if (iosfd) {
                 io_stats_dump_fd (this, iosfd);
 
-                if (iosfd->filename)
-                        GF_FREE (iosfd->filename);
+                GF_FREE (iosfd->filename);
                 GF_FREE (iosfd);
         }
 
@@ -2456,6 +2549,116 @@ io_stats_forget (xlator_t *this, inode_t *inode)
         return 0;
 }
 
+static int
+ios_init_top_stats (struct ios_conf *conf)
+{
+        int     i = 0;
+
+        GF_ASSERT (conf);
+
+        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
+                conf->list[i].iosstats = GF_CALLOC (1,
+                                         sizeof(*conf->list[i].iosstats),
+                                         gf_io_stats_mt_ios_stat);
+
+                if (!conf->list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+                LOCK_INIT (&conf->list[i].lock);
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
+                conf->thru_list[i].iosstats = GF_CALLOC (1,
+                                 sizeof (*conf->thru_list[i].iosstats),
+                                 gf_io_stats_mt_ios_stat);
+
+                if (!conf->thru_list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+                LOCK_INIT (&conf->thru_list[i].lock);
+        }
+
+        return 0;
+}
+
+static void
+ios_destroy_top_stats (struct ios_conf *conf)
+{
+        int                     i = 0;
+        struct ios_stat_head    *list_head = NULL;
+        struct ios_stat_list    *entry     = NULL;
+        struct ios_stat_list    *tmp       = NULL;
+        struct ios_stat_list    *list      = NULL;
+        struct ios_stat         *stat      = NULL;
+
+        GF_ASSERT (conf);
+
+        LOCK (&conf->lock);
+
+        conf->cumulative.nr_opens = 0;
+        conf->cumulative.max_nr_opens = 0;
+        conf->cumulative.max_openfd_time.tv_sec = 0;
+        conf->cumulative.max_openfd_time.tv_usec = 0;
+
+        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+                list_head = &conf->list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+                list_head = &conf->thru_list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        UNLOCK (&conf->lock);
+
+        return;
+}
+
+static int
+io_stats_clear (struct ios_conf *conf)
+{
+        struct timeval      now;
+        int                 ret = -1;
+
+        GF_ASSERT (conf);
+
+        if (!gettimeofday (&now, NULL))
+        {
+            LOCK (&conf->lock);
+            {
+                    ios_global_stats_clear (&conf->cumulative, &now);
+                    ios_global_stats_clear (&conf->incremental, &now);
+                    conf->increment = 0;
+            }
+            UNLOCK (&conf->lock);
+            ret = 0;
+        }
+
+        return ret;
+}
 
 int
 reconfigure (xlator_t *this, dict_t *options)
@@ -2523,7 +2726,6 @@ int
 init (xlator_t *this)
 {
         struct ios_conf    *conf = NULL;
-        int                 i = 0;
         char               *sys_log_str = NULL;
         int                 sys_log_level = -1;
         char               *log_str = NULL;
@@ -2560,35 +2762,9 @@ init (xlator_t *this)
         gettimeofday (&conf->cumulative.started_at, NULL);
         gettimeofday (&conf->incremental.started_at, NULL);
 
-        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
-                conf->list[i].iosstats = GF_CALLOC (1,
-                                         sizeof(*conf->list[i].iosstats),
-                                         gf_io_stats_mt_ios_stat);
-
-                if (!conf->list[i].iosstats) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                               "Out of memory");
-                        return -1;
-                }
-
-                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
-                LOCK_INIT (&conf->list[i].lock);
-        }
-
-	for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
-		conf->thru_list[i].iosstats = GF_CALLOC (1,
-		                 sizeof (*conf->thru_list[i].iosstats),
-				 gf_io_stats_mt_ios_stat);
-
-	        if (!conf->thru_list[i].iosstats) {
-        	        gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                	return -1;
-        	}
-
-		INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
-		LOCK_INIT (&conf->thru_list[i].lock);
-        }
+        ret = ios_init_top_stats (conf);
+        if (ret)
+                return -1;
 
         GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
 
@@ -2620,12 +2796,6 @@ void
 fini (xlator_t *this)
 {
         struct ios_conf *conf = NULL;
-        struct ios_stat_head *list_head = NULL;
-        struct ios_stat_list *entry     = NULL;
-        struct ios_stat_list *tmp       = NULL;
-        struct ios_stat_list *list      = NULL;
-        struct ios_stat      *stat      = NULL;
-        int    i                        = 0;
 
         if (!this)
                 return;
@@ -2636,45 +2806,15 @@ fini (xlator_t *this)
                 return;
         this->private = NULL;
 
-        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
-                list_head = &conf->list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        if (list)
-                                GF_FREE (list);
-                }
-        }
-
-        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
-                list_head = &conf->thru_list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        if (list)
-                                GF_FREE (list);
-                }
-        }
+        ios_destroy_top_stats (conf);
 
-        if (conf)
-                GF_FREE(conf);
+        GF_FREE(conf);
 
         gf_log (this->name, GF_LOG_INFO,
                 "io-stats translator unloaded");
         return;
 }
 
-
 int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
@@ -2682,10 +2822,11 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         struct ios_dump_args args = {0};
         dict_t       *output = NULL;
         dict_t       *dict = NULL;
-        int32_t       top_op = 0;
+        int32_t       op = 0;
         int32_t       list_cnt = 0;
         double        throughput = 0;
         double        time = 0;
+        gf_boolean_t  is_peek = _gf_false;
         va_list ap;
 
         dict = data;
@@ -2694,15 +2835,37 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         va_end (ap);
         switch (event) {
         case GF_EVENT_TRANSLATOR_INFO:
-                ret = dict_get_int32 (dict, "top-op", &top_op);
+                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+                if (ret) {
+                        ret = dict_set_int32 (output, "top-op", op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set top-op in dict");
+                                goto out;
+                        }
+                        ios_destroy_top_stats (this->private);
+                        ret = ios_init_top_stats (this->private);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to reset top stats");
+                        ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set stats-cleared"
+                                        " in dict");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "top-op", &op);
                 if (!ret) {
                         ret = dict_get_int32 (dict, "list-cnt", &list_cnt);
-                        if (top_op > IOS_STATS_TYPE_NONE &&
-                            top_op < IOS_STATS_TYPE_MAX)
+                        if (op > IOS_STATS_TYPE_NONE &&
+                            op < IOS_STATS_TYPE_MAX)
                                 ret = io_stats_dump_stats_to_dict (this, output,
-                                                             top_op, list_cnt);
-                        if (top_op == IOS_STATS_TYPE_READ_THROUGHPUT ||
-                                top_op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
+                                                             op, list_cnt);
+                        if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
+                                op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
                                 ret = dict_get_double (dict, "throughput",
                                                         &throughput);
                                 if (!ret) {
@@ -2723,9 +2886,41 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 
                         }
                 } else {
-                        (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_DICT,
-                                           output);
-                        ret = io_stats_dump (this, &args);
+                        ret = dict_get_int32 (dict, "info-op", &op);
+                        if (ret || op < GF_CLI_INFO_ALL ||
+                            GF_CLI_INFO_CLEAR < op)
+                            op = GF_CLI_INFO_ALL;
+
+                        ret = dict_set_int32 (output, "info-op", op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set info-op in dict");
+                                goto out;
+                        }
+
+                        if (GF_CLI_INFO_CLEAR == op) {
+                                ret = io_stats_clear (this->private);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to clear info stats");
+
+                                ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set stats-cleared"
+                                                " in dict");
+                        }
+                        else {
+                                ret = dict_get_str_boolean (dict, "peek",
+                                                            _gf_false);
+                                if (-1 != ret)
+                                        is_peek = ret;
+
+                                (void) ios_dump_args_init (&args,
+                                                IOS_DUMP_TYPE_DICT, output);
+                                ret = io_stats_dump (this, &args, op, is_peek);
+                        }
                 }
                 break;
         default:
@@ -2777,6 +2972,9 @@ struct xlator_fops fops = {
         .fxattrop    = io_stats_fxattrop,
         .setattr     = io_stats_setattr,
         .fsetattr    = io_stats_fsetattr,
+	.fallocate   = io_stats_fallocate,
+	.discard     = io_stats_discard,
+        .zerofill    = io_stats_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -2820,7 +3018,7 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR,
           .default_value = "CRITICAL",
           .description = "Gluster's syslog log-level",
-          .value = { "WARNING", "ERROR", "CRITICAL"}
+          .value = { "WARNING", "ERROR", "INFO", "CRITICAL"}
         },
         { .key = {"brick-log-level"},
           .type = GF_OPTION_TYPE_STR,
diff --git a/xlators/debug/trace/src/Makefile.am b/xlators/debug/trace/src/Makefile.am
index 0f1679a04..7b2597b4d 100644
--- a/xlators/debug/trace/src/Makefile.am
+++ b/xlators/debug/trace/src/Makefile.am
@@ -2,13 +2,15 @@
 xlator_LTLIBRARIES = trace.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-trace_la_LDFLAGS = -module -avoidversion
+trace_la_LDFLAGS = -module -avoid-version
 
 trace_la_SOURCES = trace.c
 trace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = trace.h trace-mem-types.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/trace/src/trace-mem-types.h b/xlators/debug/trace/src/trace-mem-types.h
new file mode 100644
index 000000000..9fa7d97c2
--- /dev/null
+++ b/xlators/debug/trace/src/trace-mem-types.h
@@ -0,0 +1,21 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __TRACE_MEM_TYPES_H__
+#define __TRACE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_trace_mem_types_ {
+        gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
+        gf_trace_mt_end
+};
+#endif
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index 09c58bdfd..c9d839356 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -1,26 +1,15 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "trace.h"
+#include "trace-mem-types.h"
 
 /**
  * xlators/debug/trace :
@@ -29,1203 +18,1388 @@
  *    Very helpful translator for debugging.
  */
 
-#include <time.h>
-#include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "common-utils.h"
-
-
-struct {
-        char *name;
-        int enabled;
-} trace_fop_names[GF_FOP_MAXVALUE];
-
-int trace_log_level = GF_LOG_INFO;
-
-static char *
-trace_stat_to_str (struct iatt *buf)
+int
+dump_history_trace (circular_buffer_t *cb, void *data)
 {
-        char    *statstr           = NULL;
-        char     atime_buf[256]    = {0,};
-        char     mtime_buf[256]    = {0,};
-        char     ctime_buf[256]    = {0,};
-        int      asprint_ret_value = 0;
-        uint64_t ia_time           = 0;
-
-        if (!buf) {
-                statstr = NULL;
-                goto out;
-        }
-
-        ia_time = buf->ia_atime;
-        strftime (atime_buf, 256, "[%b %d %H:%M:%S]",
-                  localtime ((time_t *)&ia_time));
-
-        ia_time = buf->ia_mtime;
-        strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",
-                  localtime ((time_t *)&ia_time));
+        char    *string      = NULL;
+        struct  tm      *tm  = NULL;
+        char    timestr[256] = {0,};
 
-        ia_time = buf->ia_ctime;
-        strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",
-                  localtime ((time_t *)&ia_time));
+        string = (char *)cb->data;
+        tm    = localtime (&cb->tv.tv_sec);
 
-        asprint_ret_value = gf_asprintf (&statstr,
-                                         "gfid=%s ino=%"PRIu64", mode=%o, "
-                                         "nlink=%"GF_PRI_NLINK", uid=%u, "
-                                         "gid=%u, size=%"PRIu64", "
-                                         "blocks=%"PRIu64", atime=%s, "
-                                         "mtime=%s, ctime=%s",
-                                         uuid_utoa (buf->ia_gfid), buf->ia_ino,
-                                         st_mode_from_ia (buf->ia_prot,
-                                                          buf->ia_type),
-                                         buf->ia_nlink, buf->ia_uid,
-                                         buf->ia_gid, buf->ia_size,
-                                         buf->ia_blocks, atime_buf,
-                                         mtime_buf, ctime_buf);
+        /* Since we are continuing with adding entries to the buffer even when
+           gettimeofday () fails, it's safe to check tm and then dump the time
+           at which the entry was added to the buffer */
+        if (tm) {
+                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
+                gf_proc_dump_write ("TIME", "%s", timestr);
+        }
 
-        if (asprint_ret_value < 0)
-                statstr = NULL;
+        gf_proc_dump_write ("FOP", "%s\n", string);
 
-out:
-        return statstr;
+        return 0;
 }
 
-
 int
 trace_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, fd_t *fd,
                   inode_t *inode, struct iatt *buf,
-                  struct iatt *preparent, struct iatt *postparent)
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_CREATE].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
-                                "*stbuf {%s}, *preparent {%s}, *postparent = "
-                                "{%s})", frame->root->unique,
-                                uuid_utoa (inode->gfid), op_ret, fd,
-                                statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
+                                  "*stbuf {%s}, *preparent {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid), op_ret, fd,
+                                  statstr, preparentstr, postparentstr);
 
                         /* for 'release' log */
                         fd_ctx_set (fd, this, 0);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd)
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t      *conf = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, *fd=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno, fd);
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, "
+                          "*fd=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          fd);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         /* for 'release' log */
         if (op_ret >= 0)
                 fd_ctx_set (fd, this, 0);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        TRACE_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
-
 int
 trace_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STAT].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref)
+                 int32_t count, struct iatt *buf, struct iobref *iobref,
+                 dict_t *xdata)
 {
-        char  *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READ].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             buf, iobref);
+out:
+        TRACE_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
+                            buf, iobref, xdata);
         return 0;
 }
 
-
 int
 trace_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf)
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
+        char         preopstr[4096]  = {0, };
+        char         postopstr[4096] = {0, };
+        trace_conf_t *conf           = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_WRITE].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
-
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
-
-
 int
 trace_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                   dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (readdir, frame, op_ret, op_errno, buf, xdata);
 
         return 0;
 }
 
-
 int
 trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                    dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char        string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf);
+                LOG_ELEMENT (conf, string);
+        }
 
+out:
+        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *prebuf, struct iatt *postbuf)
+                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s}",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s}",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
 
         return 0;
 }
 
-
 int
 trace_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
-                   struct iatt *statpre, struct iatt *statpost)
+                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (statpre);
-                        postopstr = trace_stat_to_str (statpost);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre, statpost);
+out:
+        TRACE_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
+                            statpost, xdata);
         return 0;
 }
 
-
 int
 trace_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost)
+                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (statpre);
-                        postopstr = trace_stat_to_str (statpost);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno,
-                             statpre, statpost);
+out:
+        TRACE_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                            statpre, statpost, xdata);
         return 0;
 }
 
-
 int
 trace_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent)
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        char *preparentstr = NULL;
-        char *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                if (op_ret >= 0) {
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, preparentstr,
-                                postparentstr);
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  " *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
                   struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent)
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preoldparentstr = NULL;
-        char  *postoldparentstr = NULL;
-        char  *prenewparentstr = NULL;
-        char  *postnewparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preoldparentstr = trace_stat_to_str (preoldparent);
-                        postoldparentstr = trace_stat_to_str (postoldparent);
-
-                        prenewparentstr = trace_stat_to_str (prenewparent);
-                        postnewparentstr = trace_stat_to_str (postnewparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *stbuf = {%s}, "
-                                "*preoldparent = {%s}, *postoldparent = {%s}"
-                                " *prenewparent = {%s}, *postnewparent = {%s})",
-                                frame->root->unique, op_ret, statstr,
-                                preoldparentstr, postoldparentstr,
-                                prenewparentstr, postnewparentstr);
+        char           statstr[4096]          = {0, };
+        char           preoldparentstr[4096]  = {0, };
+        char           postoldparentstr[4096] = {0, };
+        char           prenewparentstr[4096]  = {0, };
+        char           postnewparentstr[4096] = {0, };
+        trace_conf_t   *conf                  = NULL;
 
-                        if (preoldparentstr)
-                                GF_FREE (preoldparentstr);
+        conf = this->private;
 
-                        if (postoldparentstr)
-                                GF_FREE (postoldparentstr);
-
-                        if (prenewparentstr)
-                                GF_FREE (prenewparentstr);
-
-                        if (postnewparentstr)
-                                GF_FREE (postnewparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preoldparent, preoldparentstr);
+                        trace_stat_to_str (postoldparent, postoldparentstr);
+                        trace_stat_to_str (prenewparent, prenewparentstr);
+                        trace_stat_to_str (postnewparent, postnewparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s}, *preoldparent = {%s},"
+                                  " *postoldparent = {%s}"
+                                  " *prenewparent = {%s}, "
+                                  "*postnewparent = {%s})",
+                                  frame->root->unique, op_ret, statstr,
+                                  preoldparentstr, postoldparentstr,
+                                  prenewparentstr, postnewparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
+
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent);
+out:
+        TRACE_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
+                            preoldparent, postoldparent,
+                            prenewparent, postnewparent, xdata);
         return 0;
 }
 
-
 int
 trace_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    const char *buf, struct iatt *stbuf)
+                    const char *buf, struct iatt *stbuf, dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
 
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+                char string[4096] = {0,};
                 if (op_ret == 0) {
-                        statstr = trace_stat_to_str (stbuf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d, buf=%s, "
-                                "stbuf = { %s })",
-                                frame->root->unique, op_ret, op_errno, buf,
-                                statstr);
-                } else
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        trace_stat_to_str (stbuf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d,"
+                                  "buf=%s, stbuf = { %s })",
+                                  frame->root->unique, op_ret, op_errno,
+                                  buf, statstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
 
-                if (statstr)
-                        GF_FREE (statstr);
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, stbuf);
+out:
+        TRACE_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, stbuf,
+                            xdata);
         return 0;
 }
 
-
 int
 trace_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
                   inode_t *inode, struct iatt *buf,
-                  dict_t *xattr, struct iatt *postparent)
+                  dict_t *xdata, struct iatt *postparent)
 {
-        char  *statstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        postparentstr = trace_stat_to_str (postparent);
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*buf {%s}, *postparent {%s}",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (postparent, postparentstr);
+                        /* print buf->ia_gfid instead of inode->gfid,
+                         * since if the inode is not yet linked to the
+                         * inode table (fresh lookup) then null gfid
+                         * will be printed.
+                         */
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*buf {%s}, *postparent {%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (buf->ia_gfid),
+                                  op_ret, statstr, postparentstr);
 
                         /* For 'forget' */
                         inode_ctx_put (inode, this, 0);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xattr, postparent);
+out:
+        TRACE_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+                            xdata, postparent);
         return 0;
 }
 
-
 int
 trace_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent)
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*stbuf = {%s}, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": op_ret=%d, op_errno=%d",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, op_errno=%d",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char *statstr = NULL;
-        char *preparentstr = NULL;
-        char *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*stbuf = {%s}, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                ", *stbuf = {%s}, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  ", *stbuf = {%s}, *prebuf = {%s}, "
+                                  "*postbuf = {%s} )",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
                 inode_t *inode, struct iatt *buf,
-                struct iatt *preparent, struct iatt *postparent)
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *stbuf = {%s}, "
-                                " *prebuf = {%s}, *postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                statstr, preparentstr, postparentstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (statstr)
-                                GF_FREE (statstr);
+        conf = this->private;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char  string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_LINK].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s},  *prebuf = {%s},"
+                                  " *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  statstr, preparentstr, postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char      string[4096] = {0,};
         if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd)
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char    string[4096] = {0,};
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, fd=%p",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno, fd);
-        }
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " fd=%p",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno, fd);
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         /* for 'releasedir' log */
         if (op_ret >= 0)
                 fd_ctx_set (fd, this, 0);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        TRACE_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
-
 int
 trace_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
+        char           preparentstr[4096]  = {0, };
+        char           postparentstr[4096] = {0, };
+        trace_conf_t   *conf               = NULL;
 
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                if (op_ret >= 0) {
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s}",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, preparentstr,  postparentstr);
-
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "*prebuf={%s},  *postbuf={%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr, postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf)
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
+        char           preopstr[4096]  = {0, };
+        char           postopstr[4096] = {0, };
+        trace_conf_t   *conf           = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret, preopstr,
-                                postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
+                            postbuf, xdata);
         return 0;
 }
 
-
 int
 trace_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                  dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": ({f_bsize=%lu, f_frsize=%lu, f_blocks=%"GF_PRI_FSBLK
-                                ", f_bfree=%"GF_PRI_FSBLK", f_bavail=%"GF_PRI_FSBLK", "
-                                "f_files=%"GF_PRI_FSBLK", f_ffree=%"GF_PRI_FSBLK", f_favail=%"
-                                GF_PRI_FSBLK", f_fsid=%lu, f_flag=%lu, f_namemax=%lu}) => ret=%d",
-                                frame->root->unique, buf->f_bsize, buf->f_frsize, buf->f_blocks,
-                                buf->f_bfree, buf->f_bavail, buf->f_files, buf->f_ffree,
-                                buf->f_favail, buf->f_fsid, buf->f_flag, buf->f_namemax, op_ret);
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": ({f_bsize=%lu, "
+                                  "f_frsize=%lu, "
+                                  "f_blocks=%"GF_PRI_FSBLK
+                                  ", f_bfree=%"GF_PRI_FSBLK", "
+                                  "f_bavail=%"GF_PRI_FSBLK", "
+                                  "f_files=%"GF_PRI_FSBLK", "
+                                  "f_ffree=%"GF_PRI_FSBLK", "
+                                  "f_favail=%"GF_PRI_FSBLK", "
+                                  "f_fsid=%lu, f_flag=%lu, "
+                                  "f_namemax=%lu}) => ret=%d",
+                                  frame->root->unique, buf->f_bsize,
+                                  buf->f_frsize, buf->f_blocks,
+                                  buf->f_bfree, buf->f_bavail,
+                                  buf->f_files, buf->f_ffree,
+                                  buf->f_favail, buf->f_fsid,
+                                  buf->f_flag, buf->f_namemax, op_ret);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret,
+                          op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, dict=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno, dict);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
 
 int
 trace_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno);
-        }
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, dict=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno, dict);
-        }
+                char      string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
 
 int
 trace_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, "
+                          "op_errno=%d)", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf)
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *prebufstr = NULL;
-        char  *postbufstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                if (op_ret >= 0) {
-                        prebufstr = trace_stat_to_str (prebuf);
-                        postbufstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret,
-                                prebufstr, postbufstr);
+        char          prebufstr[4096]  = {0, };
+        char          postbufstr[4096] = {0, };
+        trace_conf_t  *conf            = NULL;
 
-                        if (prebufstr)
-                                GF_FREE (prebufstr);
-
-                        if (postbufstr)
-                                GF_FREE (postbufstr);
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, prebufstr);
+                        trace_stat_to_str (postbuf, postbufstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  prebufstr, postbufstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
-
 int
 trace_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                char string[4096]  = {0.};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d "
+                                  "buf=%s", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+              int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LK].enabled) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, {l_type=%d, l_whence=%d, "
-                                "l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, lock->l_type, lock->l_whence,
-                                lock->l_start, lock->l_len, lock->l_pid);
+                char      string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "{l_type=%d, l_whence=%d, "
+                                  "l_start=%"PRId64", "
+                                  "l_len=%"PRId64", l_pid=%u})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, lock->l_type, lock->l_whence,
+                                  lock->l_start, lock->l_len,
+                                  lock->l_pid);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
-        }
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
-
-
 int
 trace_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char   string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 trace_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *dict)
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
-
 int
 trace_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
-
 int
 trace_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local),op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     uint32_t weak_checksum, uint8_t *strong_checksum)
+                     uint32_t weak_checksum, uint8_t *strong_checksum,
+                     dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
         }
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum);
+out:
+        TRACE_STACK_UNWIND (rchecksum, frame, op_ret, op_errno, weak_checksum,
+                            strong_checksum, xdata);
 
         return 0;
 }
@@ -1235,35 +1409,54 @@ trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 trace_entrylk (call_frame_t *frame, xlator_t *this,
                const char *volume, loc_t *loc, const char *basename,
-               entrylk_cmd cmd, entrylk_type type)
+               entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (path=%s basename=%s, "
-                        "cmd=%s, type=%s)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        volume, loc->path, basename,
-                        ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
-                        ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          volume, loc->path, basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_entrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
-
 int
 trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-               loc_t *loc, int32_t cmd, struct gf_flock *flock)
+               loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+        char         *cmd_str  = NULL;
+        char         *type_str = NULL;
+        trace_conf_t *conf     = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_INODELK].enabled) {
+                char string[4096]  = {0,};
                 switch (cmd) {
 #if F_GETLK != F_GETLK64
                 case F_GETLK64:
@@ -1306,32 +1499,43 @@ trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                         break;
                 }
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (path=%s "
-                        "cmd=%s, type=%s, start=%llu, len=%llu, pid=%llu)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        volume, loc->path,
-                        cmd_str, type_str, (unsigned long long) flock->l_start,
-                        (unsigned long long) flock->l_len,
-                        (unsigned long long) flock->l_pid);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), volume,
+                          loc->path, cmd_str, type_str,
+                          (unsigned long long)flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_inodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock);
+                    volume, loc, cmd, flock, xdata);
         return 0;
 }
 
-
 int
 trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, int32_t cmd, struct gf_flock *flock)
+                fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char *cmd_str = NULL, *type_str = NULL;
+        char          *cmd_str  = NULL;
+        char          *type_str = NULL;
+        trace_conf_t  *conf     = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+                char  string[4096] = {0,};
                 switch (cmd) {
 #if F_GETLK != F_GETLK64
                 case F_GETLK64:
@@ -1374,233 +1578,351 @@ trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                         break;
                 }
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (fd =%p "
-                        "cmd=%s, type=%s, start=%llu, len=%llu, pid=%llu)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), volume, fd,
-                        cmd_str, type_str, (unsigned long long) flock->l_start,
-                        (unsigned long long) flock->l_len,
-                        (unsigned long long) flock->l_pid);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd =%p "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          cmd_str, type_str,
+                          (unsigned long long) flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
                 frame->local = fd->inode->gfid;
-        }
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         STACK_WIND (frame, trace_finodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock);
+                    volume, fd, cmd, flock, xdata);
         return 0;
 }
 
-
 int
 trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               gf_xattrop_flags_t flags, dict_t *dict)
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s (path=%s flags=%d)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s (path=%s flags=%d)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_xattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict);
+                    loc, flags, dict, xdata);
 
         return 0;
 }
 
-
 int
 trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                gf_xattrop_flags_t flags, dict_t *dict)
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, flags=%d",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, flags);
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fxattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict);
+                    fd, flags, dict, xdata);
 
         return 0;
 }
 
-
 int
 trace_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr_req)
+              loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char string[4096] = {0,};
                 /* TODO: print all the keys mentioned in xattr_req */
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
+                    loc, xdata);
 
         return 0;
 }
 
-
 int
-trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_STAT].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path);
+                char  string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_stat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->stat,
-                    loc);
+                    loc, xdata);
 
         return 0;
 }
 
-
 int
-trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s, size=%"GF_PRI_SIZET")",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, size);
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "size=%"GF_PRI_SIZET")", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          size);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
 
         return 0;
 }
 
-
 int
 trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             mode_t mode, dev_t dev, dict_t *params)
+             mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mode=%d dev=%"GF_PRI_DEV")",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mode, dev);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d "
+                          "umask=0%o, dev=%"GF_PRI_DEV")",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask, dev);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, params);
+                    loc, mode, dev, umask, xdata);
 
         return 0;
 }
 
-
 int
 trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dict_t *params)
+             mode_t umask, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mode=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mode);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_mkdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, xdata);
         return 0;
 }
 
-
 int
-trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flag=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          xflag);
+
                 frame->local = loc->inode->gfid;
-        }
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         STACK_WIND (frame, trace_unlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->unlink,
-                    loc);
+                    loc, xflag, xdata);
         return 0;
 }
 
-
 int
-trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rmdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags);
+                    loc, flags, xdata);
 
         return 0;
 }
 
-
 int
 trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, dict_t *params)
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s linkpath=%s, path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        linkpath, loc->path);
+                char     string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s linkpath=%s, path=%s"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), linkpath,
+                          loc->path, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, params);
+                    linkpath, loc, umask, xdata);
 
         return 0;
 }
 
-
 int
-trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        char oldgfid[50] = {0,};
-        char newgfid[50] = {0,};
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char string[4096] = {0,};
                 if (newloc->inode)
                         uuid_utoa_r (newloc->inode->gfid, newgfid);
                 else
@@ -1608,29 +1930,40 @@ trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 
                 uuid_utoa_r (oldloc->inode->gfid, oldgfid);
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": oldgfid=%s oldpath=%s --> newgfid=%s newpath=%s",
-                        frame->root->unique, oldgfid, oldloc->path, newgfid, newloc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s",
+                          frame->root->unique, oldgfid,
+                          oldloc->path, newgfid, newloc->path);
 
                 frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rename_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
 
         return 0;
 }
 
-
 int
-trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        char oldgfid[50] = {0,};
-        char newgfid[50] = {0,};
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LINK].enabled) {
+                char string[4096]  = {0,};
                 if (newloc->inode)
                         uuid_utoa_r (newloc->inode->gfid, newgfid);
                 else
@@ -1638,42 +1971,63 @@ trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 
                 uuid_utoa_r (oldloc->inode->gfid, oldgfid);
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": oldgfid=%s oldpath=%s --> newgfid=%s newpath=%s",
-                        frame->root->unique, oldgfid, oldloc->path,
-                        newgfid, newloc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s", frame->root->unique,
+                          oldgfid, oldloc->path, newgfid,
+                          newloc->path);
+
                 frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_link_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
         return 0;
 }
 
-
 int
 trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        uint64_t ia_time          = 0;
-        char     actime_str[256]  = {0,};
-        char     modtime_str[256] = {0,};
+        uint64_t     ia_time          = 0;
+        char         actime_str[256]  = {0,};
+        char         modtime_str[256] = {0,};
+        trace_conf_t *conf            = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char     string[4096]  =  {0,};
                 if (valid & GF_SET_ATTR_MODE) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s mode=%o)",
-                                frame->root->unique, uuid_utoa (loc->inode->gfid),
-                                loc->path, st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type));
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s mode=%o)",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s uid=%o, gid=%o",
-                                frame->root->unique,  uuid_utoa (loc->inode->gfid),
-                                loc->path, stbuf->ia_uid, stbuf->ia_gid);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s uid=%o,"
+                                  " gid=%o", frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, stbuf->ia_uid,
+                                  stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
@@ -1685,44 +2039,64 @@ trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                         strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
                                   localtime ((time_t *)&ia_time));
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s ia_atime=%s, ia_mtime=%s",
-                                frame->root->unique, uuid_utoa (loc->inode->gfid),
-                                loc->path, actime_str, modtime_str);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
                 frame->local = loc->inode->gfid;
         }
 
+out:
         STACK_WIND (frame, trace_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid);
+                    loc, stbuf, valid, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        uint64_t ia_time          = 0;
-        char     actime_str[256]  = {0,};
-        char     modtime_str[256] = {0,};
+        uint64_t       ia_time          = 0;
+        char           actime_str[256]  = {0,};
+        char           modtime_str[256] = {0,};
+        trace_conf_t   *conf            = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char     string[4096]  =  {0,};
                 if (valid & GF_SET_ATTR_MODE) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p, mode=%o",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid), fd,
-                                st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type));
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, mode=%o",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid), fd,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p, uid=%o, gid=%o",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid),
-                                fd, stbuf->ia_uid, stbuf->ia_gid);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, uid=%o, "
+                                  "gid=%o", frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, stbuf->ia_uid, stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
@@ -1734,256 +2108,394 @@ trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                         strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
                                   localtime ((time_t *)&ia_time));
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p ia_atime=%s, ia_mtime=%s",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid),
-                                fd, actime_str, modtime_str);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
                 frame->local = fd->inode->gfid;
         }
 
+out:
         STACK_WIND (frame, trace_fsetattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid);
+                    fd, stbuf, valid, xdata);
 
         return 0;
 }
 
-
 int
 trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                off_t offset)
+                off_t offset, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s, offset=%"PRId64"",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, offset);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "offset=%"PRId64"", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          offset);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_truncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->truncate,
-                    loc, offset);
+                    loc, offset, xdata);
 
         return 0;
 }
 
-
 int
 trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            int32_t flags, fd_t *fd, int32_t wbflags)
+            int32_t flags, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d fd=%p wbflags=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags, fd, wbflags);
+                char      string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags, fd);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_open_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
         return 0;
 }
 
-
 int
 trace_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+              int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s, fd=%p, flags=0%o mode=0%o",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, fd, flags, mode);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, fd=%p, "
+                          "flags=0%o mode=0%o umask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          fd, flags, mode, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
-
 int
 trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t offset, uint32_t flags)
+             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READ].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", "
-                        "offset=%"PRId64" flags=0%x)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"
+                          GF_PRI_SIZET"offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readv_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
-
 int
 trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
               struct iovec *vector, int32_t count,
-              off_t offset, uint32_t flags, struct iobref *iobref)
+              off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, count=%d, offset=%"PRId64
-                        " flag=0%x)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, count, offset, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, count=%d, "
+                          " offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, count,
+                          offset, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
         return 0;
 }
 
-
 int
-trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, (loc->inode)?
-                        uuid_utoa (loc->inode->gfid):"0", loc->path);
+                char  string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique, (loc->inode)?
+                          uuid_utoa (loc->inode->gfid):"0", loc->path);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_statfs_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->statfs,
-                    loc);
+                    loc, xdata);
         return 0;
 }
 
-
 int
-trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_flush_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->flush,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
-
 int
-trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s flags=%d fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), flags, fd);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), flags, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsync_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsync,
-                    fd, flags);
+                    fd, flags, xdata);
         return 0;
 }
 
-
 int
 trace_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int32_t flags)
+                loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 }
 
-
 int
 trace_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name)
+                loc_t *loc, const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s name=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, name);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 }
 
-
 int
 trace_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name)
+                   loc_t *loc, const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s name=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, name);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_removexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->removexattr,
-                    loc, name);
+                    loc, name, xdata);
 
         return 0;
 }
 
-
 int
-trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s fd=%p",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path, fd);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
-                    loc, fd);
+                    loc, fd, xdata);
         return 0;
 }
 
@@ -1991,15 +2503,27 @@ int
 trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 off_t offset, dict_t *dict)
 {
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", "
-                        "offset=%"PRId64" dict=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset, dict);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64" dict=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, dict);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readdirp_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdirp,
@@ -2008,83 +2532,128 @@ trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         return 0;
 }
 
-
 int
 trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t offset)
+               size_t size, off_t offset, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64,
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64,
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset);
+                    fd, size, offset, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsyncdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t datasync)
+                fd_t *fd, int32_t datasync, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s datasync=%d fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        datasync, fd);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s datasync=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), datasync, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsyncdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync);
+                    fd, datasync, xdata);
         return 0;
 }
 
-
 int
-trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+              dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mask=0%o",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mask);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          loc->path, mask);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_access_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
         return 0;
 }
 
-
 int32_t
 trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 int32_t len)
+                 int32_t len, dict_t *xdata)
 {
+
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s offset=%"PRId64" len=%u fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        offset, len, fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64
+                          "len=%u fd=%p", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, len, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rchecksum_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rchecksum,
-                    fd, offset, len);
+                    fd, offset, len, xdata);
 
         return 0;
 
@@ -2093,162 +2662,264 @@ trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
 int32_t
 trace_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 fd_t *fd, const char *basename, entrylk_cmd cmd,
-                entrylk_type type)
+                entrylk_type type, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (fd=%p basename=%s, "
-                        "cmd=%s, type=%s)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        volume, fd, basename,
-                        ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
-                        ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+                char      string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd=%p "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fentrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
+                    volume, fd, basename, cmd, type, xdata);
         return 0;
 
 }
 
 int32_t
 trace_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p name=%s",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, name);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p name=%s",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, name);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fgetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name);
+                    fd, name, xdata);
         return 0;
 }
 
 int32_t
 trace_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags)
+                 dict_t *dict, int32_t flags, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p flags=%d",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, flags);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, dict, flags, xdata);
         return 0;
 }
 
 int
 trace_ftruncate (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, off_t offset)
+                 fd_t *fd, off_t offset, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        offset, fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_ftruncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset);
+                    fd, offset, xdata);
 
         return 0;
 }
 
-
 int
-trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fstat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fstat,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
-
 int
 trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-          int32_t cmd, struct gf_flock *lock)
+          int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, cmd=%d, lock {l_type=%d, l_whence=%d, "
-                        "l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd,
-                        cmd, lock->l_type, lock->l_whence,
-                        lock->l_start, lock->l_len, lock->l_pid);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, cmd=%d, "
+                          "lock {l_type=%d, "
+                          "l_whence=%d, l_start=%"PRId64", "
+                          "l_len=%"PRId64", l_pid=%u})",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, cmd,
+                          lock->l_type, lock->l_whence,
+                          lock->l_start, lock->l_len, lock->l_pid);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_lk_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock);
+                    fd, cmd, lock, xdata);
         return 0;
 }
 
 int32_t
 trace_forget (xlator_t *this, inode_t *inode)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
         /* If user want to understand when a lookup happens,
            he should know about 'forget' too */
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s", uuid_utoa (inode->gfid));
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s", uuid_utoa (inode->gfid));
+
+                LOG_ELEMENT (conf, string);
         }
+
+out:
         return 0;
 }
 
-
 int32_t
 trace_releasedir (xlator_t *this, fd_t *fd)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s fd=%p", uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         return 0;
 }
 
 int32_t
 trace_release (xlator_t *this, fd_t *fd)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled ||
             trace_fop_names[GF_FOP_CREATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s fd=%p", uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
         }
+
+out:
         return 0;
 }
 
 
-
 void
 enable_all_calls (int enabled)
 {
@@ -2258,7 +2929,6 @@ enable_all_calls (int enabled)
                 trace_fop_names[i].enabled = enabled;
 }
 
-
 void
 enable_call (const char *name, int enabled)
 {
@@ -2286,6 +2956,105 @@ process_call_list (const char *list, int include)
         }
 }
 
+int32_t
+trace_dump_history (xlator_t *this)
+{
+        int ret = -1;
+        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+        trace_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("trace", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+
+        conf = this->private;
+        // Is it ok to return silently if log-history option his off?
+        if (conf && conf->log_history == _gf_true) {
+                gf_proc_dump_build_key (key_prefix, "xlator.debug.trace",
+                                        "history");
+                gf_proc_dump_add_section (key_prefix);
+                eh_dump (this->history, NULL, dump_history_trace);
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_trace_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int32_t       ret            = -1;
+        trace_conf_t    *conf           = NULL;
+        char            *includes = NULL, *excludes = NULL;
+
+        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+
+        conf = this->private;
+
+        includes = data_to_str (dict_get (options, "include-ops"));
+        excludes = data_to_str (dict_get (options, "exclude-ops"));
+
+        {
+                int i;
+                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
+                        trace_fop_names[i].enabled = 1;
+                }
+        }
+
+        if (includes && excludes) {
+                gf_log (this->name,
+                        GF_LOG_ERROR,
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
+                goto out;
+        }
+
+        if (includes)
+                process_call_list (includes, 1);
+        if (excludes)
+                process_call_list (excludes, 0);
+
+        /* Should resizing of the event-history be allowed in reconfigure?
+         * for which a new event_history might have to be allocated and the
+         * older history has to be freed.
+         */
+        GF_OPTION_RECONF ("log-file", conf->log_file, options, bool, out);
+
+        GF_OPTION_RECONF ("log-history", conf->log_history, options, bool, out);
+
+        ret = 0;
+
+out:
+        return ret;
+}
 
 int32_t
 init (xlator_t *this)
@@ -2293,6 +3062,10 @@ init (xlator_t *this)
         dict_t *options = NULL;
         char *includes = NULL, *excludes = NULL;
         char *forced_loglevel = NULL;
+        eh_t *history = NULL;
+        int  ret = -1;
+        size_t  history_size = TRACE_DEFAULT_HISTORY_SIZE;
+        trace_conf_t    *conf = NULL;
 
         if (!this)
                 return -1;
@@ -2307,6 +3080,12 @@ init (xlator_t *this)
                         "dangling volume. check volfile ");
         }
 
+        conf = GF_CALLOC (1, sizeof (trace_conf_t), gf_trace_mt_trace_conf_t);
+        if (!conf) {
+                gf_log (this->name, GF_LOG_ERROR, "cannot allocate "
+                        "xl->private");
+                return -1;
+        }
 
         options = this->options;
         includes = data_to_str (dict_get (options, "include-ops"));
@@ -2315,8 +3094,13 @@ init (xlator_t *this)
         {
                 int i;
                 for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        trace_fop_names[i].name = (gf_fop_list[i] ?
-                                                   gf_fop_list[i] : ":O");
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
                         trace_fop_names[i].enabled = 1;
                 }
         }
@@ -2324,14 +3108,43 @@ init (xlator_t *this)
         if (includes && excludes) {
                 gf_log (this->name,
                         GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and 'exclude-ops'");
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
                 return -1;
         }
+
         if (includes)
                 process_call_list (includes, 1);
         if (excludes)
                 process_call_list (excludes, 0);
 
+
+        GF_OPTION_INIT ("history-size", conf->history_size, size, out);
+
+        gf_log (this->name, GF_LOG_INFO, "history size %"GF_PRI_SIZET,
+                history_size);
+
+        GF_OPTION_INIT ("log-file", conf->log_file, bool, out);
+
+        gf_log (this->name, GF_LOG_INFO, "logging to file %s",
+                (conf->log_file == _gf_true)?"enabled":"disabled");
+
+        GF_OPTION_INIT ("log-history", conf->log_history, bool, out);
+
+        gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
+                (conf->log_history == _gf_true)?"enabled":"disabled");
+
+        history = eh_new (history_size, _gf_false, NULL);
+        if (!history) {
+                gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
+                        "initialized");
+                return -1;
+        }
+
+        this->history = history;
+
+        conf->trace_log_level = GF_LOG_INFO;
+
         if (dict_get (options, "force-log-level")) {
                 forced_loglevel = data_to_str (dict_get (options,
                                                          "force-log-level"));
@@ -2339,25 +3152,34 @@ init (xlator_t *this)
                         goto setloglevel;
 
                 if (strcmp (forced_loglevel, "INFO") == 0)
-                        trace_log_level = GF_LOG_INFO;
+                        conf->trace_log_level = GF_LOG_INFO;
                 else if (strcmp (forced_loglevel, "TRACE") == 0)
-                        trace_log_level = GF_LOG_TRACE;
+                        conf->trace_log_level = GF_LOG_TRACE;
                 else if (strcmp (forced_loglevel, "ERROR") == 0)
-                        trace_log_level = GF_LOG_ERROR;
+                        conf->trace_log_level = GF_LOG_ERROR;
                 else if (strcmp (forced_loglevel, "DEBUG") == 0)
-                        trace_log_level = GF_LOG_DEBUG;
+                        conf->trace_log_level = GF_LOG_DEBUG;
                 else if (strcmp (forced_loglevel, "WARNING") == 0)
-                        trace_log_level = GF_LOG_WARNING;
+                        conf->trace_log_level = GF_LOG_WARNING;
                 else if (strcmp (forced_loglevel, "CRITICAL") == 0)
-                        trace_log_level = GF_LOG_CRITICAL;
+                        conf->trace_log_level = GF_LOG_CRITICAL;
                 else if (strcmp (forced_loglevel, "NONE") == 0)
-                        trace_log_level = GF_LOG_NONE;
+                        conf->trace_log_level = GF_LOG_NONE;
         }
 
 setloglevel:
-        gf_log_set_loglevel (trace_log_level);
+        gf_log_set_loglevel (conf->trace_log_level);
+        this->private = conf;
+        ret = 0;
+out:
+        if (ret == -1) {
+                if (history)
+                        GF_FREE (history);
+                if (conf)
+                        GF_FREE (conf);
+        }
 
-        return 0;
+        return ret;
 }
 
 void
@@ -2366,6 +3188,9 @@ fini (xlator_t *this)
         if (!this)
                 return;
 
+        if (this->history)
+                eh_destroy (this->history);
+
         gf_log (this->name, GF_LOG_INFO,
                 "trace translator unloaded");
         return;
@@ -2414,7 +3239,6 @@ struct xlator_fops fops = {
         .fsetattr    = trace_fsetattr,
 };
 
-
 struct xlator_cbks cbks = {
         .release     = trace_release,
         .releasedir  = trace_releasedir,
@@ -2430,5 +3254,21 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR
           /*.value = { ""} */
         },
+        { .key  = {"history-size"},
+          .type = GF_OPTION_TYPE_SIZET,
+          .default_value = "1024",
+        },
+        { .key  = {"log-file"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
+        { .key  = {"log-history"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
         { .key  = {NULL} },
 };
+
+struct xlator_dumpops dumpops = {
+        .history = trace_dump_history
+};
diff --git a/xlators/debug/trace/src/trace.h b/xlators/debug/trace/src/trace.h
new file mode 100644
index 000000000..045eefb36
--- /dev/null
+++ b/xlators/debug/trace/src/trace.h
@@ -0,0 +1,98 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <time.h>
+#include <errno.h>
+#include "glusterfs.h"
+#include "xlator.h"
+#include "common-utils.h"
+#include "event-history.h"
+#include "logging.h"
+#include "circ-buff.h"
+#include "statedump.h"
+#include "options.h"
+
+#define TRACE_DEFAULT_HISTORY_SIZE 1024
+
+typedef struct {
+        /* Since the longest fop name is fremovexattr i.e 12 characters, array size
+         * is kept 24, i.e double of the maximum.
+         */
+        char name[24];
+        int enabled;
+} trace_fop_name_t;
+
+trace_fop_name_t trace_fop_names[GF_FOP_MAXVALUE];
+
+typedef struct {
+        gf_boolean_t log_file;
+        gf_boolean_t log_history;
+        size_t       history_size;
+        int trace_log_level;
+} trace_conf_t;
+
+#define TRACE_STACK_UNWIND(op, frame, params ...)                       \
+        do {                                                            \
+                frame->local = NULL;                                    \
+                STACK_UNWIND_STRICT (op, frame, params);                \
+        } while (0);
+
+#define LOG_ELEMENT(_conf, _string)                                     \
+        do {                                                            \
+                if (_conf) {                                            \
+                        if ((_conf->log_history) == _gf_true)           \
+                                gf_log_eh ("%s", _string);              \
+                        if ((_conf->log_file) == _gf_true)              \
+                                gf_log (THIS->name, _conf->trace_log_level, \
+                                        "%s", _string);                 \
+                }                                                       \
+        } while (0);
+
+#define trace_stat_to_str(buf, statstr)                                 \
+        do {                                                            \
+                char     atime_buf[256]    = {0,};                      \
+                char     mtime_buf[256]    = {0,};                      \
+                char     ctime_buf[256]    = {0,};                      \
+                uint64_t ia_time           = 0;                         \
+                                                                        \
+                if (!buf)                                               \
+                        break;                                          \
+                                                                        \
+                ia_time = buf->ia_atime;                                \
+                strftime (atime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                ia_time = buf->ia_mtime;                                \
+                strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                ia_time = buf->ia_ctime;                                \
+                strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                snprintf (statstr, sizeof (statstr),                    \
+                          "gfid=%s ino=%"PRIu64", mode=%o, "            \
+                          "nlink=%"GF_PRI_NLINK", uid=%u, "             \
+                          "gid=%u, size=%"PRIu64", "                    \
+                          "blocks=%"PRIu64", atime=%s, "                \
+                          "mtime=%s, ctime=%s",                         \
+                          uuid_utoa (buf->ia_gfid), buf->ia_ino,        \
+                          st_mode_from_ia (buf->ia_prot,                \
+                                           buf->ia_type),               \
+                          buf->ia_nlink, buf->ia_uid,                   \
+                          buf->ia_gid, buf->ia_size,                    \
+                          buf->ia_blocks, atime_buf,                    \
+                          mtime_buf, ctime_buf);                        \
+        } while (0);
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
index 2cbde680f..36efc6698 100644
--- a/xlators/encryption/Makefile.am
+++ b/xlators/encryption/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = rot-13
+SUBDIRS = rot-13 crypt
 
 CLEANFILES = 
diff --git a/xlators/cluster/unify/Makefile.am b/xlators/encryption/crypt/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/cluster/unify/Makefile.am
+++ b/xlators/encryption/crypt/Makefile.am
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
new file mode 100644
index 000000000..b13f65043
--- /dev/null
+++ b/xlators/encryption/crypt/src/Makefile.am
@@ -0,0 +1,24 @@
+if ENABLE_CRYPT_XLATOR
+
+xlator_LTLIBRARIES = crypt.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
+
+crypt_la_LDFLAGS = -module -avoid-version -lssl -lcrypto
+
+crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
+crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+else
+
+noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+endif
+\ No newline at end of file
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
new file mode 100644
index 000000000..1ec41495c
--- /dev/null
+++ b/xlators/encryption/crypt/src/atom.c
@@ -0,0 +1,962 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/*
+ *                              Glossary
+ *
+ *
+ * cblock            (or cipher block). A logical unit in a file.
+ *                   cblock size is defined as the number of bits
+ *                   in an input (or output) block of the block
+ *                   cipher (*). Cipher block size is a property of
+ *                   cipher algorithm. E.g. cblock size is 64 bits
+ *                   for DES, 128 bits for AES, etc.
+ *
+ * atomic cipher     A cipher algorithm, which requires some chunks of
+ * algorithm         text to be padded at left and(or) right sides before
+ *                   cipher transaform.
+ *
+ *
+ * block (atom)      Minimal chunk of file's data, which doesn't require
+ *                   padding. We'll consider logical units in a file of
+ *                   block size (atom size).
+ *
+ * cipher algorithm  Atomic cipher algorithm, which requires the last
+ * with EOF issue    incomplete cblock in a file to be padded with some
+ *                   data (usually zeros).
+ *
+ *
+ * operation, which  reading/writing from offset, which is not aligned to
+ * forms a gap at    to atom size
+ * the beginning
+ *
+ *
+ * operation, which  reading/writing count bytes starting from offset off,
+ * forms a gap at    so that off+count is not aligned to atom_size
+ * the end
+ *
+ * head block        the first atom affected by an operation, which forms
+ *                   a gap at the beginning, or(and) at the end.
+ *                   Сomment. Head block has at least one gap (either at
+ *                   the beginning, or at the end)
+ *
+ *
+ * tail block        the last atom different from head, affected by an
+ *                   operation, which forms a gap at the end.
+ *                   Сomment: Tail block has exactly one gap (at the end).
+ *
+ *
+ * partial block     head or tail block
+ *
+ *
+ * full block        block without gaps.
+ *
+ *
+ * (*) Recommendation for Block Cipher Modes of Operation
+ *     Methods and Techniques
+ *     NIST Special Publication 800-38A Edition 2001
+ */
+
+/*
+ * atom->offset_at()
+ */
+static off_t offset_at_head(struct avec_config *conf)
+{
+	return conf->aligned_offset;
+}
+
+static off_t offset_at_hole_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_hole_conf(frame));
+}
+
+static off_t offset_at_data_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_data_conf(frame));
+}
+
+
+static off_t offset_at_tail(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0) +
+		(conf->nr_full_blocks << get_atom_bits(object));
+}
+
+static off_t offset_at_hole_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_hole_conf(frame), object);
+}
+
+
+static off_t offset_at_data_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_data_conf(frame), object);
+}
+
+static off_t offset_at_full(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0);
+}
+
+static off_t offset_at_data_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_data_conf(frame), object);
+}
+
+static off_t offset_at_hole_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->io_size_nopad()
+ */
+
+static uint32_t io_size_nopad_head(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	uint32_t gap_at_beg;
+	uint32_t gap_at_end;
+
+	check_head_block(conf);
+
+	gap_at_beg = conf->off_in_head;
+
+	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
+		gap_at_end = 0;
+	else
+		gap_at_end = get_atom_size(object) - conf->off_in_tail;
+
+	return get_atom_size(object) - (gap_at_beg + gap_at_end);
+}
+
+static uint32_t io_size_nopad_tail(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	check_tail_block(conf);
+	return conf->off_in_tail;
+}
+
+static uint32_t io_size_nopad_full(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{	
+	check_full_block(conf);
+	return get_atom_size(object);
+}
+
+static uint32_t io_size_nopad_data_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_hole_conf(frame), object);
+}
+
+static uint32_t offset_in_head(struct avec_config *conf)
+{
+	check_cursor_head(conf);
+
+	return conf->off_in_head;
+}
+
+static uint32_t offset_in_tail(call_frame_t *frame,
+			       struct object_cipher_info *object)
+{
+	return 0;
+}
+
+static uint32_t offset_in_full(struct avec_config *conf,
+			       struct object_cipher_info *object)
+{
+	check_cursor_full(conf);
+
+	if (has_head_block(conf))
+		return (conf->cursor - 1) << get_atom_bits(object);
+	else
+		return conf->cursor << get_atom_bits(object);
+}
+
+static uint32_t offset_in_data_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_data_conf(frame));
+}
+
+static uint32_t offset_in_hole_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_hole_conf(frame));
+}
+
+static uint32_t offset_in_data_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_data_conf(frame), object);
+}
+
+static uint32_t offset_in_hole_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->rmw()
+ */
+/*
+ * Pre-conditions:
+ * @vec contains plain text of the latest
+ * version.
+ *
+ * Uptodate gaps of the @partial block with
+ * this plain text, encrypt the whole block
+ * and write the result to disk.
+ */
+static int32_t rmw_partial_block(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 struct iovec *vec,
+				 int32_t count,
+				 struct iatt *stbuf,
+				 struct iobref *iobref,
+				 struct rmw_atom *atom)
+{
+	size_t was_read = 0;
+	uint64_t file_size;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *partial = atom->get_iovec(frame, 0);
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_partial_block;
+#if DEBUG_CRYPT
+	gf_boolean_t check_last_cblock = _gf_false;
+#endif
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto exit;
+
+	file_size = local->cur_file_size;
+	was_read = op_ret;
+
+	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
+		/*
+		 * head atom with a non-uptodate gap
+		 * at the beginning
+		 *
+		 * fill the gap with plain text of the
+		 * latest version. Convert a part of hole
+		 * (if any) to zeros.
+		 */
+		int32_t i;
+		int32_t copied = 0;
+		int32_t to_gap; /* amount of data needed to uptodate
+				   the gap at the beginning */
+#if 0
+		int32_t hole = 0; /* The part of the hole which
+				   * got in the head block */
+#endif /* 0 */
+		to_gap = conf->off_in_head;
+
+		if (was_read < to_gap) {
+			if (file_size >
+			    offset_at_head(conf) + was_read) {
+				/*
+				 * It is impossible to uptodate
+				 * head block: too few bytes have
+				 * been read from disk, so that
+				 * partial write is impossible.
+				 *
+				 * It could happen because of many
+				 * reasons: IO errors, (meta)data
+				 * corruption in the local file system,
+				 * etc.
+				 */
+				gf_log(this->name, GF_LOG_WARNING,
+				     "Can not uptodate a gap at the beginning");
+				local->op_ret = -1;
+				local->op_errno = EIO;
+				goto exit;
+			}
+#if 0
+			hole = to_gap - was_read;
+#endif /* 0 */
+			to_gap = was_read;
+		}
+		/*
+		 * uptodate the gap at the beginning
+		 */
+		for (i = 0; i < count && copied < to_gap; i++) {
+			int32_t to_copy;
+
+			to_copy = vec[i].iov_len;
+			if (to_copy > to_gap - copied)
+				to_copy = to_gap - copied;
+
+			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
+			copied += to_copy;
+		}
+#if 0
+		/*
+		 * If possible, convert part of the
+		 * hole, which got in the head block
+		 */
+		ret = TRY_LOCK(&local->hole_lock);
+		if (!ret) {
+			if (local->hole_handled)
+				/*
+				 * already converted by
+				 * crypt_writev_cbk()
+				 */
+				UNLOCK(&local->hole_lock);
+			else {
+				/*
+				 * convert the part of the hole
+				 * which got in the head block
+				 * to zeros.
+				 *
+				 * Update the orig_offset to make
+				 * sure writev_cbk() won't care
+				 * about this part of the hole.
+				 *
+				 */
+				memset(partial->iov_base + to_gap, 0, hole);
+
+				conf->orig_offset -= hole;
+				conf->orig_size += hole;
+				UNLOCK(&local->hole_lock);
+			}
+		}
+		else /*
+		      * conversion is being performed
+		      * by crypt_writev_cbk()
+		      */
+			;
+#endif /* 0 */
+	}
+	if (atom->locality == TAIL_ATOM ||
+	    (!has_tail_block(conf) && conf->off_in_tail)) {
+		/*
+		 * tail atom, or head atom with a non-uptodate
+		 * gap at the end.
+		 *
+		 * fill the gap at the end of the block
+		 * with plain text of the latest version.
+		 * Pad the result, (if needed)
+		 */
+		int32_t i;
+		int32_t to_gap;
+		int copied;
+		off_t off_in_tail;
+		int32_t to_copy;
+
+		off_in_tail = conf->off_in_tail;
+		to_gap = conf->gap_in_tail;
+
+		if (to_gap && was_read < off_in_tail + to_gap) {
+			/*
+			 * It is impossible to uptodate
+			 * the gap at the end: too few bytes
+			 * have been read from disk, so that
+			 * partial write is impossible.
+			 *
+			 * It could happen because of many
+			 * reasons: IO errors, (meta)data
+			 * corruption in the local file system,
+			 * etc.
+			 */
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Can not uptodate a gap at the end");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * uptodate the gap at the end
+		 */
+		copied = 0;
+		to_copy = to_gap;
+		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
+			uint32_t from_vec, off_in_vec;
+
+			off_in_vec = 0;
+			from_vec = vec[i].iov_len;
+			if (from_vec > to_copy) {
+				off_in_vec = from_vec - to_copy;
+				from_vec = to_copy;
+			}
+			memcpy(partial->iov_base +
+			       off_in_tail + to_gap - copied - from_vec,
+			       vec[i].iov_base + off_in_vec,
+			       from_vec);
+
+			gf_log(this->name, GF_LOG_DEBUG,
+			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
+			       (int)from_vec,
+			       (int)off_in_tail + to_gap - copied - from_vec,
+			       (int)off_in_vec);
+
+			copied += from_vec;
+			to_copy -= from_vec;
+		}
+		partial->iov_len = off_in_tail + to_gap;
+
+		if (object_alg_should_pad(object)) {
+			int32_t resid = 0;
+			resid = partial->iov_len & (object_alg_blksize(object) - 1);
+			if (resid) {
+				/*
+				 * append a new EOF padding
+				 */
+				local->eof_padding_size =
+					object_alg_blksize(object) - resid;
+
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "set padding size %d",
+				       local->eof_padding_size);
+
+				memset(partial->iov_base + partial->iov_len,
+				       1,
+				       local->eof_padding_size);
+				partial->iov_len += local->eof_padding_size;
+#if DEBUG_CRYPT
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "pad cblock with %d zeros:",
+				       local->eof_padding_size);
+				dump_cblock(this,
+					    (unsigned char *)partial->iov_base +
+					    partial->iov_len - object_alg_blksize(object));
+				check_last_cblock = _gf_true;
+#endif
+			}
+		}
+	}
+	/*
+	 * encrypt the whole block
+	 */
+	encrypt_aligned_iov(object,
+			    partial,
+			    1,
+			    atom->offset_at(frame, object));
+#if DEBUG_CRYPT
+	if (check_last_cblock == _gf_true) {
+		gf_log(this->name, GF_LOG_DEBUG,
+		       "encrypt last cblock with offset %llu",
+		       (unsigned long long)atom->offset_at(frame, object));
+		dump_cblock(this, (unsigned char *)partial->iov_base +
+			    partial->iov_len - object_alg_blksize(object));
+	}
+#endif
+	set_local_io_params_writev(frame, object, atom,
+				   atom->offset_at(frame, object),
+				   iovec_get_size(partial, 1));
+	/*
+	 * write the whole block to disk
+	 */
+	end_writeback_partial_block = dispatch_end_writeback(local->fop);
+	conf->cursor ++;
+	STACK_WIND(frame,
+		   end_writeback_partial_block,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->writev,
+		   local->fd,
+		   partial,
+		   1,
+		   atom->offset_at(frame, object),
+		   local->flags,
+		   local->iobref_data,
+		   local->xdata);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "submit partial block: %d bytes from %d offset",
+	       (int)iovec_get_size(partial, 1),
+	       (int)atom->offset_at(frame, object));
+ exit:
+	return 0;
+}
+
+/*
+ * Perform a (read-)modify-write sequence.
+ * This should be performed only after approval
+ * of upper server-side manager, i.e. the caller
+ * needs to make sure this is his turn to rmw.
+ */
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype)
+{
+	int32_t ret;
+	dict_t *dict;
+	struct rmw_atom *atom;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	atom = atom_by_types(local->active_setup, ltype);
+	/*
+	 * To perform the "read" component of the read-modify-write
+	 * sequence the crypt translator does stack_wind to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 */
+	dict = dict_new();
+	if (!dict) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		return;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		dict_unref(dict);
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   atom->rmw,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   fd,
+		   atom->count_to_uptodate(frame, object), /* count */
+		   atom->offset_at(frame, object), /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	dict_unref(dict);
+}
+
+/*
+ * submit blocks of FULL_ATOM type
+ */
+void submit_full(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
+	uint32_t count; /* total number of full blocks to submit */
+	uint32_t granularity; /* number of blocks to submit in one iteration */
+
+	uint64_t off_in_file; /* start offset in the file, bytes */
+	uint32_t off_in_atom; /* start offset in the atom, blocks */
+	uint32_t blocks_written = 0; /* blocks written for this submit */
+
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_full_block;
+	/*
+	 * Write full blocks by groups of granularity size.
+	 */
+	end_writeback_full_block = dispatch_end_writeback(local->fop);
+
+	if (is_ordered_mode(frame)) {
+		uint32_t skip = has_head_block(conf) ? 1 : 0;
+		count = 1;
+		granularity = 1;
+		/*
+		 * calculate start offset using cursor value;
+		 * here we should take into accout head block,
+		 * which corresponds to cursor value 0.
+		 */
+		off_in_file = atom->offset_at(frame, object) +
+			((conf->cursor - skip) << get_atom_bits(object));
+		off_in_atom = conf->cursor - skip;
+	}
+	else {
+		/*
+		 * in parallel mode
+		 */
+		count = conf->nr_full_blocks;
+		granularity = MAX_IOVEC;
+		off_in_file = atom->offset_at(frame, object);
+		off_in_atom = 0;
+	}
+	while (count) {
+		uint32_t blocks_to_write = count;
+
+		if (blocks_to_write > granularity)
+			blocks_to_write = granularity;
+		if (conf->type == HOLE_ATOM)
+			/*
+			 * reset iovec before encryption
+			 */
+			memset(atom->get_iovec(frame, 0)->iov_base,
+			       0,
+			       get_atom_size(object));
+		/*
+		 * encrypt the group
+		 */
+		encrypt_aligned_iov(object,
+				    atom->get_iovec(frame,
+						    off_in_atom +
+						    blocks_written),
+				    blocks_to_write,
+				    off_in_file + (blocks_written <<
+						   get_atom_bits(object)));
+		
+		set_local_io_params_writev(frame, object, atom,
+		        off_in_file + (blocks_written << get_atom_bits(object)),
+			blocks_to_write <<  get_atom_bits(object));
+
+		conf->cursor += blocks_to_write;
+
+		STACK_WIND(frame,
+			   end_writeback_full_block,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev,
+			   local->fd,
+			   atom->get_iovec(frame, off_in_atom + blocks_written),
+			   blocks_to_write,
+			   off_in_file + (blocks_written << get_atom_bits(object)),
+			   local->flags,
+			   local->iobref_data ? local->iobref_data : local->iobref,
+			   local->xdata);
+
+		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
+		       blocks_to_write,
+		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
+
+		count -= blocks_to_write;
+		blocks_written += blocks_to_write;
+	}
+	return;
+}
+
+static int32_t rmw_data_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_data_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, TAIL_ATOM));
+}
+
+static int32_t rmw_hole_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_hole_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
+}
+
+/*
+ * atom->count_to_uptodate()
+ */
+static uint32_t count_to_uptodate_head(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	if (conf->acount == 1 && conf->off_in_tail)
+		return get_atom_size(object);
+	else
+		/* there is no need to read the whole head block */
+		return conf->off_in_head;
+}
+
+static uint32_t count_to_uptodate_tail(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	/* we need to read the whole tail block */
+	return get_atom_size(object);
+}
+
+static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_hole_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_hole_conf(frame), object);
+}
+
+/* atom->get_config() */
+
+static struct avec_config *get_config_data(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->data_conf;
+}
+
+static struct avec_config *get_config_hole(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->hole_conf;
+}
+
+/*
+ * atom->get_iovec()
+ */
+static struct iovec *get_iovec_hole_head(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec;
+}
+
+static struct iovec *get_iovec_hole_full(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0);
+}
+
+static inline struct iovec *get_iovec_hole_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->blocks_in_pool - 1);
+}
+
+static inline struct iovec *get_iovec_data_head(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec;
+}
+
+static inline struct iovec *get_iovec_data_full(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
+}
+
+static inline struct iovec *get_iovec_data_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec +
+		(conf->off_in_head ? 1 : 0) +
+		conf->nr_full_blocks;
+}
+
+static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
+	[DATA_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_data_head,
+	  .offset_at = offset_at_data_head,
+	  .offset_in = offset_in_data_head,
+	  .get_iovec = get_iovec_data_head,
+	  .io_size_nopad = io_size_nopad_data_head,
+	  .count_to_uptodate = count_to_uptodate_data_head,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_data_tail,
+	  .offset_at = offset_at_data_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_data_tail,
+	  .io_size_nopad = io_size_nopad_data_tail,
+	  .count_to_uptodate = count_to_uptodate_data_tail,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_data_full,
+	  .offset_in = offset_in_data_full,
+	  .get_iovec = get_iovec_data_full,
+	  .io_size_nopad = io_size_nopad_data_full,
+	  .get_config = get_config_data
+	},
+	[HOLE_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_hole_head,
+	  .offset_at = offset_at_hole_head,
+	  .offset_in = offset_in_hole_head,
+	  .get_iovec = get_iovec_hole_head,
+	  .io_size_nopad = io_size_nopad_hole_head,
+	  .count_to_uptodate = count_to_uptodate_hole_head,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_hole_tail,
+	  .offset_at = offset_at_hole_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_hole_tail,
+	  .io_size_nopad = io_size_nopad_hole_tail,
+	  .count_to_uptodate = count_to_uptodate_hole_tail,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_hole_full,
+	  .offset_in = offset_in_hole_full,
+	  .get_iovec = get_iovec_hole_full,
+	  .io_size_nopad = io_size_nopad_hole_full,
+	  .get_config = get_config_hole
+	}
+};
+
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality)
+{
+	return &atoms[data][locality];
+}
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
new file mode 100644
index 000000000..7c212ad5d
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-common.h
@@ -0,0 +1,141 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_COMMON_H__
+#define __CRYPT_COMMON_H__
+
+#define INVAL_SUBVERSION_NUMBER (0xff)
+#define CRYPT_INVAL_OP   (GF_FOP_NULL)
+
+#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
+#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
+#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
+#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
+#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
+#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
+#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
+
+
+/* messages for crypt_open() */
+#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
+#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
+
+#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
+
+#define noop   do {; } while (0)
+#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+
+/*
+ * Format of file's metadata
+ */
+struct crypt_format {
+	uint8_t loader_id;  /* version of metadata loader */
+	uint8_t versioned[0]; /* file's metadata of specific version */
+} __attribute__((packed));
+
+typedef enum {
+	AES_CIPHER_ALG,
+	LAST_CIPHER_ALG
+} cipher_alg_t;
+
+typedef enum {
+	XTS_CIPHER_MODE,
+	LAST_CIPHER_MODE
+} cipher_mode_t;
+
+typedef enum {
+	MTD_LOADER_V1,
+	LAST_MTD_LOADER
+} mtd_loader_id;
+
+static inline void msgflags_set_mtd_rlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline void msgflags_set_mtd_wlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
+{
+	return  msgflags_check_mtd_rlock(flags) ||
+		msgflags_check_mtd_wlock(flags);
+}
+
+/*
+ * returns number of logical blocks occupied
+ * (maybe partially) by @count bytes
+ * at offset @start.
+ */
+static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
+					    int blkbits)
+{
+	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
+}
+
+/*
+ * are two bytes (represented by offsets @off1
+ * and @off2 respectively) in the same logical
+ * block.
+ */
+static inline int in_same_lblock(uint64_t off1, uint64_t off2,
+				 int blkbits)
+{
+	return off1 >> blkbits == off2 >> blkbits;
+}
+
+static inline void dump_cblock(xlator_t *this, unsigned char *buf)
+{
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
+	       (buf)[0],
+	       (buf)[1],
+	       (buf)[2],
+	       (buf)[3],
+	       (buf)[4],
+	       (buf)[5],
+	       (buf)[6],
+	       (buf)[7],
+	       (buf)[8],
+	       (buf)[9],
+	       (buf)[10],
+	       (buf)[11],
+	       (buf)[12],
+	       (buf)[13],
+	       (buf)[14],
+	       (buf)[15]);
+}
+
+#endif /* __CRYPT_COMMON_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
new file mode 100644
index 000000000..2eab921fc
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-mem-types.h
@@ -0,0 +1,44 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __CRYPT_MEM_TYPES_H__
+#define __CRYPT_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_crypt_mem_types_ {
+	gf_crypt_mt_priv = gf_common_mt_end + 1,
+	gf_crypt_mt_inode,
+	gf_crypt_mt_data,
+	gf_crypt_mt_mtd,
+	gf_crypt_mt_loc,
+	gf_crypt_mt_iatt,
+	gf_crypt_mt_key,
+	gf_crypt_mt_iovec,
+	gf_crypt_mt_char,
+	gf_crypt_mt_end,
+};
+
+#endif /* __CRYPT_MEM_TYPES_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
+
+
+
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
new file mode 100644
index 000000000..1abdad31d
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.c
@@ -0,0 +1,4522 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#include <ctype.h>
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master);
+static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				       uint64_t from, off_t size);
+static int32_t load_file_size(call_frame_t *frame, void *cookie,
+			      xlator_t *this, int32_t op_ret, int32_t op_errno,
+			      dict_t *dict, dict_t *xdata);
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype);
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype);
+static void put_one_call_open(call_frame_t *frame);
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
+static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
+static void free_avec_data(crypt_local_t *local);
+static void free_avec_hole(crypt_local_t *local);
+
+static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
+					glusterfs_fop_t fop)
+{
+	crypt_local_t *local = NULL;
+
+	local = mem_get0(this->local_pool);
+	if (!local) {
+		gf_log(this->name, GF_LOG_ERROR, "out of memory");
+		return NULL;
+	}
+	local->fop = fop;
+	LOCK_INIT(&local->hole_lock);
+	LOCK_INIT(&local->call_lock);
+	LOCK_INIT(&local->rw_count_lock);
+
+	frame->local = local;
+	return local;
+}
+
+struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
+{
+	int ret;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+
+	ret = inode_ctx_get(inode, this, &value);
+	if (ret == -1) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not get inode info");
+		return NULL;
+	}
+	info = (struct crypt_inode_info *)(long)value;
+	if (info == NULL) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not obtain inode info");
+		return NULL;
+	}
+	return info;
+}
+
+static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
+						      xlator_t *this)
+{
+	if (local->info)
+		return local->info;
+	local->info = get_crypt_inode_info(local->fd->inode, this);
+	return local->info;
+}
+
+static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
+						 loc_t *loc)
+{
+	struct crypt_inode_info *info;
+
+	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
+	if (!info) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		gf_log ("crypt", GF_LOG_WARNING,
+			"Can not allocate inode info");
+		return NULL;
+	}
+	memset(info, 0, sizeof(*info));
+#if DEBUG_CRYPT
+	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!info->loc) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
+		GF_FREE(info);
+		return NULL;
+	}
+	if (loc_copy(info->loc, loc)){
+		GF_FREE(info->loc);
+		GF_FREE(info);
+		return NULL;
+	}
+#endif /* DEBUG_CRYPT */
+
+	local->info = info;
+	return info;
+}
+
+static void free_inode_info(struct crypt_inode_info *info)
+{
+#if DEBUG_CRYPT
+	loc_wipe(info->loc);
+	GF_FREE(info->loc);
+#endif
+	memset(info, 0, sizeof(*info));
+	GF_FREE(info);
+}
+
+int crypt_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        if (!inode_ctx_del (inode, this, &ctx_addr))
+                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
+        return 0;
+}
+
+#if DEBUG_CRYPT
+static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
+		       struct iovec *vec, int32_t count, struct iatt *stbuf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = get_object_cinfo(local->info);
+	struct avec_config *conf = &local->data_conf;
+	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
+
+	if (read <= 0)
+		return;
+	if (read != iovec_get_size(vec, count))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"op_ret differs from amount of read bytes");
+
+	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (!= 0 mod(cblock size))");
+
+	if (conf->aligned_offset + read >
+	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (too large))");
+
+}
+
+#define PT_BYTES_TO_DUMP (32)
+static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
+{
+	int32_t to_dump;
+	char str[PT_BYTES_TO_DUMP + 1];
+
+	if (!avec)
+		return;
+	to_dump = avec->iov_len;
+	if (to_dump > PT_BYTES_TO_DUMP)
+		to_dump = PT_BYTES_TO_DUMP;
+	memcpy(str, avec->iov_base, to_dump);
+	memset(str + to_dump, '0', 1);
+	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
+}
+
+static int32_t data_conf_invariant(struct avec_config *conf)
+{
+	return conf->acount ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		conf->nr_full_blocks;
+}
+
+static int32_t hole_conf_invariant(struct avec_config *conf)
+{
+	return conf->blocks_in_pool ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		!!has_full_blocks(conf);
+}
+
+static void crypt_check_conf(struct avec_config *conf)
+{
+	int32_t ret = 0;
+	const char *msg;
+
+	switch (conf->type) {
+	case DATA_ATOM:
+		msg = "data";
+		ret = data_conf_invariant(conf);
+		break;
+	case HOLE_ATOM:
+		msg = "hole";
+		ret = hole_conf_invariant(conf);
+		break;
+	default:
+		msg = "unknown";
+	}
+	if (!ret)
+		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
+}
+
+static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	uint64_t local_file_size;
+
+	switch(local->fop) {
+	case GF_FOP_FTRUNCATE:
+		return;
+	case GF_FOP_WRITE:
+		local_file_size = local->new_file_size;
+		break;
+	case GF_FOP_READ:
+		if (parent_is_crypt_xlator(frame, this))
+			return;
+		local_file_size = local->cur_file_size;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
+		return;
+	}
+	if (buf->ia_size != round_up(local_file_size,
+					 object_alg_blksize(object)))
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "bad ia_size in buf (%llu), should be %llu",
+		       (unsigned long long)buf->ia_size,
+		       (unsigned long long)round_up(local_file_size,
+						    object_alg_blksize(object)));
+}
+
+#else
+#define check_read(frame, this, op_ret, vec, count, stbuf) noop
+#define dump_plain_text(local, avec) noop
+#define crypt_check_conf(conf) noop
+#define check_buf(frame, this, buf) noop
+#endif /* DEBUG_CRYPT */
+
+/*
+ * Pre-conditions:
+ * @vec represents a ciphertext of expanded size and
+ * aligned offset.
+ *
+ * Compound a temporal vector @avec with block-aligned
+ * components, decrypt and fix it up to represent a chunk
+ * of data corresponding to the original size and offset.
+ * Pass the result to the next translator.
+ */
+int32_t crypt_readv_cbk(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			struct iovec *vec,
+			int32_t count,
+			struct iatt *stbuf,
+			struct iobref *iobref,
+			dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *avec;
+	uint32_t i;
+	uint32_t to_vec;
+	uint32_t to_user;
+
+	check_buf(frame, this, stbuf);
+	check_read(frame, this, op_ret, vec, count, stbuf);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->iobref = iobref_ref(iobref);
+
+	local->buf = *stbuf;
+	local->buf.ia_size = local->cur_file_size;
+
+	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
+		goto put_one_call;
+
+	if (conf->orig_offset >= local->cur_file_size) {
+		local->op_ret = 0;
+		goto put_one_call;
+	}
+	/* 
+	 * correct config params with real file size
+	 * and actual amount of bytes read
+	 */
+	set_config_offsets(frame, this,
+			   conf->orig_offset, op_ret, DATA_ATOM, 0);
+
+	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
+		conf->orig_size = local->cur_file_size - conf->orig_offset;
+	/*
+	 * calculate amount of data to be returned
+	 * to user.
+	 */
+	to_user = op_ret;
+	if (conf->aligned_offset + to_user <= conf->orig_offset) {
+		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	to_user -= (conf->aligned_offset - conf->orig_offset);
+
+	if (to_user > conf->orig_size)
+		to_user = conf->orig_size;
+	local->rw_count = to_user;
+
+	op_errno = set_config_avec_data(this, local,
+					conf, object, vec, count);
+	if (op_errno) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
+		goto put_one_call;
+	}
+	avec = conf->avec;
+#if DEBUG_CRYPT
+	if (conf->off_in_tail != 0 &&
+	    conf->off_in_tail < object_alg_blksize(object) &&
+	    object_alg_should_pad(object))
+		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
+		       conf->off_in_tail);
+	if (iovec_get_size(vec, count) != 0 &&
+	    in_same_lblock(conf->orig_offset + iovec_get_size(vec, count) - 1,
+			   local->cur_file_size - 1,
+			   object_alg_blkbits(object))) {
+		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
+		dump_cblock(this,
+			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
+			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
+		dump_cblock(this,
+			    (unsigned char *)(vec[count - 1].iov_base) +
+			    vec[count - 1].iov_len - object_alg_blksize(object));
+	}
+#endif
+	decrypt_aligned_iov(object, avec,
+			    conf->acount, conf->aligned_offset);
+	/*
+	 * pass proper plain data to user
+	 */
+	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
+	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
+
+	to_vec = to_user;
+	for (i = 0; i < conf->acount; i++) {
+		if (avec[i].iov_len > to_vec)
+			avec[i].iov_len = to_vec;
+		to_vec -= avec[i].iov_len;
+	}
+ put_one_call:
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t do_readv(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			dict_t *dict,
+			dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->cur_file_size = data_to_uint64(data);
+	
+	get_one_call(frame);
+	STACK_WIND(frame,
+		   crypt_readv_cbk,
+		   FIRST_CHILD (this),
+		   FIRST_CHILD (this)->fops->readv,
+		   local->fd,
+		   /*
+		    * FIXME: read amount can be reduced
+		    */
+		   local->data_conf.expanded_size,
+		   local->data_conf.aligned_offset,
+		   local->flags,
+		   local->xdata);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call(frame);
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size
+	 */
+	STACK_WIND(frame,
+		   do_readv,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return  0;
+ error:
+	fd_unref(local->fd);
+	if (local->xdata)
+		dict_unref(local->xdata);
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    op_errno,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t readv_trivial_completion(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					struct iatt *buf,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "stat failed (%d)", op_errno);
+		goto error;
+	}
+	local->buf = *buf;
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;	
+ error:
+	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
+			    NULL, 0, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t crypt_readv(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    size_t size,
+		    off_t offset,
+		    uint32_t flags, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
+	       (int)size, (long long)offset);
+	if (parent_is_crypt_xlator(frame, this))
+		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
+#endif	
+	local = crypt_alloc_local(frame, this, GF_FOP_READ);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	if (size == 0)
+		goto trivial;
+
+	local->fd = fd_ref(fd);
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	set_config_offsets(frame, this, offset, size,
+			   DATA_ATOM, 0);
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * We are called by crypt_writev (or cypt_ftruncate)
+		 * to perform the "read" component of the read-modify-write
+		 * (or read-prune-write) sequence for some atom;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 *
+		 * Retrieve current file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		get_one_call(frame);
+		STACK_WIND(frame,
+			   crypt_readv_cbk,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->readv,
+			   local->fd,
+			   /*
+			    * FIXME: read amount can be reduced
+			    */
+			   local->data_conf.expanded_size,
+			   local->data_conf.aligned_offset,
+			   flags,
+			   NULL);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_readv_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   readv_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom,
+				off_t io_offset,
+				uint32_t io_size)
+{
+	crypt_local_t *local = frame->local;
+
+	local->io_offset = io_offset;
+	local->io_size = io_size;
+
+	local->io_offset_nopad =
+		atom->offset_at(frame, object) + atom->offset_in(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad offset to %llu",
+	       (unsigned long long)local->io_offset_nopad);
+
+	local->io_size_nopad = atom->io_size_nopad(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad size to %llu",
+	       (unsigned long long)local->io_size_nopad);
+
+	local->update_disk_file_size = 0;
+	/*
+	 * NOTE: eof_padding_size is 0 for all full atoms;
+	 * For head and tail atoms it will be set up at rmw_partial block()
+	 */
+	local->new_file_size = local->cur_file_size;
+
+	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
+
+		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
+
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "set new file size to %llu",
+		       (unsigned long long)local->new_file_size);
+
+		local->update_disk_file_size = 1;
+	}
+}
+
+void set_local_io_params_ftruncate(call_frame_t *frame,
+				   struct object_cipher_info *object)
+{
+	uint32_t resid;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
+	if (resid) {
+		local->eof_padding_size =
+			object_alg_blksize(object) - resid;
+		local->new_file_size = conf->aligned_offset;
+		local->update_disk_file_size = 0;
+		/*
+		 * file size will be updated
+		 * in the ->writev() stack,
+		 * when submitting file tail
+		 */
+	}
+	else {	
+		local->eof_padding_size = 0;
+		local->new_file_size = conf->orig_offset;
+		local->update_disk_file_size = 1;
+		/*
+		 * file size will be updated
+		 * in this ->ftruncate stack
+		 */
+	}
+}
+
+static inline void submit_head(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, HEAD_ATOM);
+}
+
+static inline void submit_tail(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, TAIL_ATOM);
+}
+
+static void submit_hole(call_frame_t *frame, xlator_t *this)
+{
+	/*
+	 * hole conversion always means
+	 * appended write and goes in ordered fashion
+	 */
+	do_ordered_submit(frame, this, HOLE_ATOM);
+}
+
+static void submit_data(call_frame_t *frame, xlator_t *this)
+{
+	if (is_ordered_mode(frame)) {
+		do_ordered_submit(frame, this, DATA_ATOM);
+		return;
+	}
+	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
+	get_nr_calls(frame, nr_calls_data(frame));
+	do_parallel_submit(frame, this, DATA_ATOM);
+	return;
+}
+
+/*
+ * heplers called by writev_cbk, fruncate_cbk in ordered mode
+ */
+
+static inline int32_t should_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	return conf->avec != NULL;
+}
+
+static inline int32_t should_resume_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
+		/*
+		 * Don't submit a part of hole, which
+		 * fits into a data block:
+ 		 * this part of hole will be converted
+		 * as a gap filled by zeros in data head
+		 * block.
+		 */
+		return conf->cursor < conf->acount - 1;
+	else
+		return conf->cursor < conf->acount;
+}
+
+static inline int32_t should_resume_submit_data(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	if (is_ordered_mode(frame))
+		return conf->cursor < conf->acount;
+	/*
+	 * parallel writes
+	 */
+	return 0;
+}
+
+static inline int32_t should_submit_data_after_hole(crypt_local_t *local)
+{
+	return local->data_conf.avec != NULL;
+}
+
+static void update_local_file_params(call_frame_t *frame,
+				     xlator_t *this,
+				     struct iatt *prebuf,
+				     struct iatt *postbuf)
+{
+	crypt_local_t *local = frame->local;
+
+	check_buf(frame, this, postbuf);
+
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	local->prebuf.ia_size  = local->cur_file_size;
+	local->postbuf.ia_size = local->new_file_size;
+
+	local->cur_file_size = local->new_file_size;	
+}
+
+static int32_t end_writeback_writev(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    struct iatt *prebuf,
+				    struct iatt *postbuf,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret <= 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "writev iteration failed");
+		goto put_one_call;
+	}
+	/*
+	 * op_ret includes paddings (atom's head, atom's tail and EOF)
+	 */
+	if (op_ret < local->io_size) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Incomplete writev iteration");
+		goto put_one_call;
+	}
+	op_ret -= local->eof_padding_size;
+	local->op_ret = op_ret;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local)) {
+
+		LOCK(&local->rw_count_lock);
+		local->rw_count += op_ret;
+		UNLOCK(&local->rw_count_lock);
+
+		if (should_resume_submit_data(frame))
+			submit_data(frame, this);
+	}
+	else {
+		/*
+		 * hole conversion is going on;
+		 * don't take into account written zeros
+		 */
+		if (should_resume_submit_hole(local))
+			submit_hole(frame, this);
+
+		else if (should_submit_data_after_hole(local))
+			submit_data(frame, this);
+	}
+ put_one_call:
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+#define crypt_writev_cbk end_writeback_writev
+
+#define HOLE_WRITE_CHUNK_BITS 12
+#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
+
+/*
+ * Convert hole of size @size at offset @off to
+ * zeros and prepare respective iovecs for submit.
+ * The hole lock should be held.
+ *
+ * Pre-conditions:
+ * @local->file_size is set and valid.
+ */
+int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				uint64_t off, off_t size)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
+
+	ret = set_config_avec_hole(this, local,
+				   &local->hole_conf, object, local->fop);
+	crypt_check_conf(&local->hole_conf);
+
+	return ret;
+}
+
+/*
+ * prepare for submit @count bytes at offset @from
+ */
+int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
+				off_t from, int32_t size, struct iovec *vec,
+				int32_t vec_count, int32_t setup_gap)
+{
+	uint32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, from, size,
+			   DATA_ATOM, setup_gap);
+
+	ret = set_config_avec_data(this, local,
+				   &local->data_conf, object, vec, vec_count);
+	crypt_check_conf(&local->data_conf);
+
+	return ret;
+}
+
+static void free_avec(struct iovec *avec,
+		      char **pool, int blocks_in_pool)
+{
+	if (!avec)
+		return;
+	GF_FREE(pool);
+	GF_FREE(avec);
+}
+
+static void free_avec_data(crypt_local_t *local)
+{
+	return free_avec(local->data_conf.avec,
+			 local->data_conf.pool,
+			 local->data_conf.blocks_in_pool);
+}
+
+static void free_avec_hole(crypt_local_t *local)
+{
+	return free_avec(local->hole_conf.avec,
+			 local->hole_conf.pool,
+			 local->hole_conf.blocks_in_pool);
+}
+
+
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (has_head_block(conf))
+		submit_head(frame, this);
+
+	if (has_full_blocks(conf))
+		submit_full(frame, this);
+
+	if (has_tail_block(conf))
+		submit_tail(frame, this);
+	return;
+}
+
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (should_submit_head_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_head(frame, this);
+	}
+	else if (should_submit_full_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_full(frame, this);
+	}
+	else if (should_submit_tail_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_tail(frame, this);
+	}
+	else
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "nothing has been submitted in ordered mode");
+	return;
+}
+
+static int32_t do_writev(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
+
+	if (local->cur_file_size < local->data_conf.orig_offset) {
+		/*
+		 * Set up hole config
+		 */
+		op_errno = prepare_for_submit_hole(frame,
+			       this,
+			       local->cur_file_size,
+			       local->data_conf.orig_offset - local->cur_file_size);
+		if (op_errno) {
+			local->op_ret = -1;
+			local->op_errno = op_errno;
+			goto error;
+		}
+	}
+	if (should_submit_hole(local))
+		submit_hole(frame, this);
+	else 
+		submit_data(frame, this);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_writev,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;	
+}
+
+static int32_t writev_trivial_completion(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 struct iatt *buf,
+					 dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+	
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+int crypt_writev(call_frame_t *frame,
+		 xlator_t *this,
+		 fd_t *fd,
+		 struct iovec *vec,
+		 int32_t count,
+		 off_t offset,
+		 uint32_t flags,
+		 struct iobref *iobref,
+		 dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+#if DEBUG_CRYPT
+	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
+		(int)iovec_get_size(vec, count), (long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	if (iobref)
+		local->iobref = iobref_ref(iobref);
+	/*
+	 * to update real file size on the server
+	 */
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (iovec_get_size(vec, count) == 0)
+		goto trivial;
+
+	ret = prepare_for_submit_data(frame, this, offset,
+				      iovec_get_size(vec, count),
+				      vec, count, 0 /* don't setup gup
+						       in tail: we don't
+						       know file size yet */);
+	if (ret)
+		goto error;
+
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * we are called by shinking crypt_ftruncate(),
+		 * which doesn't perform hole conversion;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 */
+
+		/*
+		 * extract file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		submit_data(frame, this);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	/*
+	 * lock the file and retrieve its size
+	 */
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_writev_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   writev_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->iobref)
+		iobref_unref(iobref);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	set_config_offsets(frame, this,
+			   offset,
+			   0, /* count */
+			   DATA_ATOM,
+			   0 /* since we prune, there is no
+				gap in tail to uptodate */);
+	return 0;
+}
+
+/*
+ * Finish the read-prune-modify sequence
+ *
+ * Can be invoked as
+ * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
+ * 2) ->writev_cbk() for non-cblock-aligned prune
+ */
+
+static int32_t prune_complete(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      struct iatt *prebuf,
+			      struct iatt *postbuf,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as ->ftruncate_cbk()
+ *
+ * Perform the "write" component of the
+ * read-prune-write sequence.
+ *
+ * submuit the rest of the file
+ */
+static int32_t prune_submit_file_tail(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno,
+				      struct iatt *prebuf,
+				      struct iatt *postbuf,
+				      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	dict_t *dict;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	dict = dict_new();
+	if (!dict) {
+		op_errno = ENOMEM;
+		goto error;
+	}
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+	local->new_file_size = conf->orig_offset;
+
+	/*
+	 * The rest of the file is a partial block and, hence,
+	 * should be written via RMW sequence, so the crypt xlator
+	 * does STACK_WIND to itself.
+	 *
+	 * Pass current file size to crypt_writev()
+	 */
+	op_errno = dict_set(dict,
+			    FSIZE_XATTR_PREFIX,
+			    data_from_uint64(local->cur_file_size));
+	if (op_errno) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "can not set key to update file size");
+		dict_unref(dict);
+		goto error;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "passing current file size (%llu) to crypt_writev",
+	       (unsigned long long)local->cur_file_size);
+	/*
+	 * Padding will be filled with
+	 * zeros by rmw_partial_block()
+	 */
+	STACK_WIND(frame,
+		   prune_complete,
+		   this,
+		   this->fops->writev, /* crypt_writev */
+		   local->fd,
+		   &local->vec,
+		   1,
+		   conf->aligned_offset, /* offset to write from */
+		   0,
+		   local->iobref,
+		   dict);
+
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as a callback of ->writev() invoked in behalf
+ * of ftruncate(): it can be
+ * 1) ordered writes issued by hole conversion in the case of
+ *    expanded truncate, or
+ * 2) an rmw partial data block issued by non-cblock-aligned
+ * prune.
+ */
+int32_t end_writeback_ftruncate(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				struct iatt *prebuf,
+				struct iatt *postbuf,
+				dict_t *xdata)
+{
+	crypt_local_t  *local = frame->local;
+	/*
+	 * if nothing has been written,
+	 * then it must be an error
+	 */
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local))
+		/* case (2) */
+		goto put_one_call;
+	/* case (1) */
+	if (should_resume_submit_hole(local))
+		submit_hole(frame, this);
+	/*
+	 * case of hole, when we should't resume
+	 */
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform prune and write components of the
+ * read-prune-write sequence.
+ *
+ * Called as ->readv_cbk()
+ *
+ * Pre-conditions:
+ * @vec contains the latest atom of the file
+ * (plain text)
+ */
+static int32_t prune_write(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno,
+			   struct iovec *vec,
+			   int32_t count,
+			   struct iatt *stbuf,
+			   struct iobref *iobref,
+			   dict_t *xdata)
+{
+	int32_t i;
+	size_t to_copy;
+	size_t copied = 0;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret == -1)
+		goto put_one_call;
+
+	/*
+	 * At first, uptodate head block
+	 */
+	if (iovec_get_size(vec, count) < conf->off_in_head) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Failed to uptodate head block for prune");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	local->vec.iov_len = conf->off_in_head;
+	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
+					gf_crypt_mt_data);
+
+	if (local->vec.iov_base == NULL) {
+	        gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to calloc head block for prune");
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+	        goto put_one_call;
+	}
+	for (i = 0; i < count; i++) {
+		to_copy = vec[i].iov_len;
+		if (to_copy > local->vec.iov_len - copied)
+			to_copy = local->vec.iov_len - copied;
+
+		memcpy((char *)local->vec.iov_base + copied,
+		       vec[i].iov_base,
+		       to_copy);
+		copied += to_copy;
+		if (copied == local->vec.iov_len)
+			break;
+	}
+	/*
+	 * perform prune with aligned offset
+	 * (i.e. at this step we prune a bit
+	 * more then it is needed
+	 */
+	STACK_WIND(frame,
+		   prune_submit_file_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->ftruncate,
+		   local->fd,
+		   conf->aligned_offset,
+		   local->xdata);
+	return 0;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform a read-prune-write sequence
+ */
+int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
+{
+	int32_t ret = 0;
+	dict_t *dict = NULL;
+	crypt_local_t  *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_local_io_params_ftruncate(frame, object);
+	get_one_call_nolock(frame);
+
+	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
+		/*
+		 * cblock-aligned prune:
+		 * we don't need read and write components,
+		 * just cut file body
+		 */
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "prune without RMW (at offset %llu",
+		       (unsigned long long)conf->orig_offset);
+
+		STACK_WIND(frame,
+			   prune_complete,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->ftruncate,
+			   local->fd,
+			   conf->orig_offset,
+			   local->xdata);
+		return 0;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "prune with RMW (at offset %llu",
+	       (unsigned long long)conf->orig_offset);	
+	/*
+	 * We are about to perform the "read" component of the
+	 * read-prune-write sequence. It means that we need to
+	 * read encrypted data from disk and decrypt it.
+	 * So, the crypt translator does STACK_WIND to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 
+	 */
+	dict = dict_new();
+	if (!dict) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		ret = ENOMEM;
+		goto exit;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   prune_write,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   local->fd,
+		   get_atom_size(object), /* bytes to read */
+		   conf->aligned_offset, /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	if (dict)
+		dict_unref(dict);
+	return ret;
+}
+
+/*
+ * File prune is more complicated than expand.
+ * First we need to read the latest atom to not lose info
+ * needed for proper update. Also we need to make sure that
+ * every component of read-prune-write sequence leaves data
+ * consistent
+ *
+ * Non-cblock aligned prune is performed as read-prune-write
+ * sequence:
+ *
+ * 1) read the latest atom;
+ * 2) perform cblock-aligned prune
+ * 3) issue a write request for the end-of-file
+ */
+int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	int32_t ret;
+
+	ret = prepare_for_prune(frame, this, offset);
+	if (ret)
+		return ret;
+	return read_prune_write(frame, this);
+}
+
+int32_t expand_file(call_frame_t *frame, xlator_t *this,
+		    uint64_t offset)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+
+	ret = prepare_for_submit_hole(frame, this,
+				      local->old_file_size,
+				      offset - local->old_file_size);
+	if (ret)
+		return ret;
+	submit_hole(frame, this);
+	return 0;
+}
+
+static int32_t ftruncate_trivial_completion(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    struct iatt *buf,
+					    dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t do_ftruncate(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    dict_t *dict,
+			    dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	if (local->data_conf.orig_offset == local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "trivial ftruncate (current file size %llu)",
+		       (unsigned long long)local->cur_file_size);
+#endif
+		goto trivial;
+	}
+	else if (local->data_conf.orig_offset < local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = prune_file(frame,
+				      this,
+				      local->data_conf.orig_offset);
+	}
+	else {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = expand_file(frame,
+				       this,
+				       local->data_conf.orig_offset);
+	}
+	if (op_errno)
+		goto error;
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   ftruncate_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	/*
+	 * finish with ftruncate
+	 */
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_ftruncate,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * ftruncate is performed in 2 steps:
+ * . recieve file size;
+ * . expand or prune file.
+ */
+static int32_t crypt_ftruncate(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       off_t offset,
+			       dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	local->data_conf.orig_offset = offset;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_ftruncate_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+/* ->flush_cbk() */
+int32_t truncate_end(call_frame_t *frame,
+		     void *cookie,
+		     xlator_t *this,
+		     int32_t op_ret,
+		     int32_t op_errno,
+		     dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_UNWIND_STRICT(truncate,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local->xdata);
+	return 0;
+}
+
+/* ftruncate_cbk() */
+int32_t truncate_flush(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iatt *prebuf,
+		       struct iatt *postbuf,
+		       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *fd = local->fd;
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	STACK_WIND(frame,
+		   truncate_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   fd,
+		   NULL);
+	fd_unref(fd);
+	return 0;
+}
+
+/*
+ * is called as ->open_cbk()
+ */
+static int32_t truncate_begin(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0) {
+		fd_unref(fd);
+		STACK_UNWIND_STRICT(truncate,
+				    frame,
+				    op_ret,
+				    op_errno, NULL, NULL, NULL);
+		return 0;
+	}
+	/*
+	 * crypt_truncate() is implemented via crypt_ftruncate(),
+	 * so the crypt xlator does STACK_WIND to itself here
+	 */
+	STACK_WIND(frame,
+		   truncate_flush,
+		   this,
+		   this->fops->ftruncate, /* crypt_ftruncate */
+		   fd,
+		   local->offset,
+		   NULL);
+	return 0;
+}
+
+/*
+ * crypt_truncate() is implemented via crypt_ftruncate() as a
+ * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
+ */
+int32_t crypt_truncate(call_frame_t *frame,
+		       xlator_t *this,
+		       loc_t *loc,
+		       off_t offset,
+		       dict_t *xdata)
+{
+	fd_t *fd;
+	crypt_local_t *local;
+
+#if DEBUG_CRYPT
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "truncate file %s at offset %llu",
+	       loc->path, (unsigned long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
+	if (!local)
+		goto error;
+
+	fd = fd_create(loc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->offset = offset;
+	local->xdata = xdata;
+	STACK_WIND(frame,
+		   truncate_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   loc,
+		   O_RDWR,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
+	return 0;
+}
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_WRITE:
+		return end_writeback_writev;
+	case GF_FOP_FTRUNCATE:
+		return end_writeback_ftruncate;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
+		return NULL;
+	}
+}
+
+/*
+ * true, if the caller needs metadata string
+ */
+static int32_t is_custom_mtd(dict_t *xdata)
+{
+	data_t *data;
+	uint32_t flags;
+
+	if (!xdata)
+		return 0;
+
+	data = dict_get(xdata, MSGFLAGS_PREFIX);
+	if (!data)
+		return 0;
+	if (data->len != sizeof(uint32_t)) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "Bad msgflags size (%d)", data->len);
+		return -1;
+	}
+	flags = *((uint32_t *)data->data);
+	return msgflags_check_mtd_lock(&flags);
+}
+
+static int32_t crypt_open_done(call_frame_t *frame,
+			       void *cookie,
+			       xlator_t *this,
+			       int32_t op_ret,
+			       int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret < 0)
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+	put_one_call_open(frame);
+	return 0;
+}
+
+static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_open_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+}
+
+/*
+ * load private inode info at open time
+ * called as ->fgetxattr_cbk()
+ */
+static int load_mtd_open(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	int32_t ret;
+	gf_boolean_t upload_info;
+	data_t *mtd;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	/*
+	 * first, check for cached info
+	 */
+	ret = inode_ctx_get(local->fd->inode, this, &value);
+	if (ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Inode info expected, but not found");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * info has been found in the cache
+		 */
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info hasn't been found in the cache.
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			goto exit;
+		}
+		init_inode_info_head(info, local->fd);
+		upload_info = _gf_true;
+	}
+	/*
+	 * extract metadata
+	 */
+	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
+	if (!mtd) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		gf_log (this->name, GF_LOG_WARNING,
+			"Format string wasn't found");
+		goto exit;
+	}
+	/*
+	 * authenticate metadata against the path
+	 */	
+	ret = open_format((unsigned char *)mtd->data,
+			  mtd->len,
+			  local->loc,
+			  info,
+			  get_master_cinfo(priv),
+			  local,
+			  upload_info);
+	if (ret) {
+		local->op_ret = -1;
+		local->op_errno = ret;
+		goto exit;
+	}
+	if (upload_info) {
+		ret = init_inode_info_tail(info, get_master_cinfo(priv));
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;	
+		}
+		ret = inode_ctx_put(local->fd->inode,
+				    this, (uint64_t)(long)info);
+		if (ret == -1) {
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+	}
+	if (local->custom_mtd) {
+		/*
+		 * pass the metadata string to the customer
+		 */
+		ret = dict_set_static_bin(local->xdata,
+					  CRYPTO_FORMAT_PREFIX,
+					  mtd->data,
+					  mtd->len);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;
+		}
+	}
+ exit:
+	if (!local->custom_mtd)
+		crypt_open_tail(frame, this);
+	else
+		put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
+				       void *cookie,
+				       xlator_t *this,
+				       int32_t op_ret,
+				       int32_t op_errno,
+				       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   load_mtd_open,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   CRYPTO_FORMAT_PREFIX,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+/*
+ * verify metadata against the specified pathname
+ */
+static int32_t crypt_open_cbk(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	else if (local->custom_mtd){
+		local->xdata = dict_new();
+		if (!local->xdata) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			gf_log ("crypt", GF_LOG_ERROR,
+				"Can not get new dict for mtd string");
+			goto exit;
+		}
+	}
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_open_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc,
+			  int32_t flags,
+			  fd_t *fd,
+			  dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc) {
+		ret = ENOMEM;
+		goto error;
+	}
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	ret = is_custom_mtd(xdata);
+	if (ret < 0) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+		ret = EINVAL;
+		goto error;
+	}
+	local->custom_mtd = ret;
+
+	if ((flags & O_ACCMODE) == O_WRONLY)
+		/*
+		 * we can't open O_WRONLY, because
+		 * we need to do read-modify-write
+		 */
+		flags = (flags & ~O_ACCMODE) | O_RDWR;
+	/*
+	 * Make sure that out translated offsets
+	 * and counts won't be ignored
+	 */
+	flags &= ~O_APPEND;
+	get_one_call_nolock(frame);
+	STACK_WIND(frame,
+		   crypt_open_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->open,
+		   loc,
+		   flags,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(open,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct object_cipher_info *object = &info->cinfo;
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
+	       uuid_utoa(info->oid));
+#endif
+	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
+									master);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * Init inode info at ->create() time
+ */
+static void init_inode_info_create(struct crypt_inode_info *info,
+				   struct master_cipher_info *master,
+				   data_t *data)
+{
+	struct object_cipher_info *object;
+
+	info->nr_minor = CRYPT_XLATOR_ID;
+	memcpy(info->oid, data->data, data->len);
+
+	object = &info->cinfo;
+
+	object->o_alg        = master->m_alg;
+	object->o_mode       = master->m_mode;
+	object->o_block_bits = master->m_block_bits;
+	object->o_dkey_size  = master->m_dkey_size;
+}
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
+{
+	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
+}
+
+static int32_t crypt_create_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno, dict_t *xdata)
+{
+	crypt_private_t *priv = this->private;
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0) {
+		free_inode_info(info);
+		goto unwind;
+	}
+	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
+	if (op_errno) {
+		op_ret = -1;
+		free_inode_info(info);
+		goto unwind;
+	}
+	/*
+	 * FIXME: drop major subversion number
+	 */
+	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
+	if (op_ret == -1) {
+		op_errno = EIO;
+		free_inode_info(info);
+		goto unwind;
+	}
+ unwind:
+	free_format(local);
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int crypt_create_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	dict_unref(local->xattr);
+
+	if (op_ret < 0)
+		goto error;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_create_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	free_inode_info(local->info);
+	free_format(local);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+
+	STACK_WIND(frame,
+		   crypt_create_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   local->fd,
+		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+		   0,
+		   NULL);
+	return 0;
+ error: 
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+	if (local->xdata)
+		dict_unref(local->xdata);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * Create and store crypt-specific format on disk;
+ * Populate cache with private inode info
+ */
+static int32_t crypt_create_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				fd_t *fd,
+				inode_t *inode,
+				struct iatt *buf,
+				struct iatt *preparent,
+				struct iatt *postparent,
+				dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);	
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND(frame,
+		   crypt_create_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);	
+	return 0;
+ error:
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_create(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t flags,
+			    mode_t mode,
+			    mode_t umask,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	int ret;
+	data_t *data;
+	crypt_local_t *local;
+	crypt_private_t *priv;
+	struct master_cipher_info *master;
+	struct crypt_inode_info *info;
+
+	priv = this->private;
+	master = get_master_cinfo(priv);
+
+	if (master_alg_atomic(master)) {
+		/*
+		 * We can't open O_WRONLY, because we
+		 * need to do read-modify-write.
+		 */
+		if ((flags & O_ACCMODE) == O_WRONLY)
+			flags = (flags & ~O_ACCMODE) | O_RDWR;
+		/*
+		 * Make sure that out translated offsets
+		 * and counts won't be ignored
+		 */
+		flags &= ~O_APPEND;
+	}
+	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	data = dict_get(xdata, "gfid-req");
+	if (!data) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
+		goto error;
+	}
+	if (data->len != sizeof(uuid_t)) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad gfid size (%d), should be %d",
+		       (int)data->len, (int)sizeof(uuid_t));
+		goto error;
+	}
+	info = alloc_inode_info(local, loc);
+	if (!info){
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * NOTE:
+	 * format has to be created BEFORE
+	 * proceeding to the untrusted server
+	 */
+	ret = alloc_format_create(local);
+	if (ret) {
+		free_inode_info(info);
+		goto error;
+	}
+	init_inode_info_create(info, master, data);
+
+	ret = create_format(local->format,
+			    loc,
+			    info,
+			    master);
+	if (ret) {
+		free_inode_info(info);
+		goto error;		
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		free_inode_info(info);
+		free_format(local);	
+		goto error;
+	}
+	ret = dict_set_static_bin(local->xattr,
+				  CRYPTO_FORMAT_PREFIX,
+				  local->format,
+				  new_format_size());
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame,
+		   crypt_create_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->create,
+		   loc,
+		   flags,
+		   mode,
+		   umask,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	gf_log("crypt", GF_LOG_WARNING, "can not create file");
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    -1,
+			    ret,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+/*
+ * FIXME: this should depends on the version of format string
+ */
+static int32_t filter_crypt_xattr(dict_t *dict,
+				  char *key, data_t *value, void *data)
+{
+        dict_del(dict, key);
+        return 0;
+}
+
+static int32_t crypt_fsetxattr(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       dict_t *dict,
+			       int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_fsetxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   fd,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * TBD: verify file metadata before wind
+ */
+static int32_t crypt_setxattr(call_frame_t *frame,
+			      xlator_t *this,
+			      loc_t *loc,
+			      dict_t *dict,
+			      int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_setxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   loc,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t linkop_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0 &&
+	    op_errno == ENOENT &&
+	    local->loc->inode->ia_type == IA_IFLNK) {
+		local->op_ret = 0;
+		local->op_errno = 0;
+	}
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * unpin inode on the server
+ */
+static int32_t link_flush(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  inode_t *inode,
+			  struct iatt *buf,
+			  struct iatt *preparent,
+			  struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	link_unwind(frame);
+	return 0;
+}
+
+void link_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	inode_t *inode;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(link,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	inode = local->inode;
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc) {
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(link,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (inode)
+		inode_unref(inode);
+}
+
+void link_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   link_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->link,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+/*
+ * unlink()
+ */
+static int32_t unlink_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *preparent,
+			    struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unlink_unwind(frame);
+	return 0;
+}
+
+void unlink_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(unlink,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(unlink,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+}
+
+void unlink_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   unlink_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->unlink,
+		   local->loc,
+		   local->flags,
+		   local->xdata);
+}
+
+void rename_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(rename,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);	
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	prenewparent = local->prenewparent;
+	postnewparent = local->postnewparent;	
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc){
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(rename,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    prenewparent,
+			    postnewparent,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (prenewparent)
+		GF_FREE(prenewparent);
+	if (postnewparent)
+		GF_FREE(postnewparent);
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t rename_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	rename_unwind(frame);
+	return 0;
+}
+
+static int32_t rename_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *buf,
+			    struct iatt *preoldparent,
+			    struct iatt *postoldparent,
+			    struct iatt *prenewparent,
+			    struct iatt *postnewparent,
+			    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	dict_unref(local->xdata);
+	local->xdata = NULL;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	local->buf = *buf;
+	local->prebuf = *preoldparent;
+	local->postbuf = *postoldparent;
+	if (prenewparent) {
+		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
+						gf_crypt_mt_iatt);
+		if (!local->prenewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->prenewparent = *prenewparent;
+	}
+	if (postnewparent) {
+		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
+						 gf_crypt_mt_iatt);
+		if (!local->postnewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->postnewparent = *postnewparent;
+	}
+	STACK_WIND(frame,
+		   rename_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	rename_unwind(frame);
+	return 0;
+}
+
+void rename_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   rename_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->rename,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+static int32_t __do_linkop(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_wind_handler_t wind_fn;
+	linkop_unwind_handler_t unwind_fn;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret >= 0)
+		wind_fn(frame, this);
+	else {
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+		unwind_fn(frame);
+	}
+	return 0;
+}
+
+static int32_t do_linkop(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if(op_ret < 0)
+		goto error;
+
+	lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:		
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * Update the metadata string (against the new pathname);
+ * submit the result
+ */
+static int32_t linkop_begin(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	gf_boolean_t upload_info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+	struct crypt_inode_info *info;
+	data_t *old_mtd;
+	uint32_t new_mtd_size;
+	uint64_t value = 0;
+	void (*unwind_fn)(call_frame_t *frame);
+	void (*wind_fn)(call_frame_t *frame, xlator_t *this);
+	mtd_op_t mop;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	mop = linkop_mtdop_dispatch(local->fop);
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto wind;
+	if (op_ret < 0)
+		/*
+		 * verification failed
+		 */
+		goto error;
+
+	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
+	if (!old_mtd) {
+		op_errno = EIO;
+		gf_log (this->name, GF_LOG_DEBUG,
+			"Metadata string wasn't found");
+		goto error;
+	}
+	new_mtd_size = format_size(mop, old_mtd->len);
+	op_errno = alloc_format(local, new_mtd_size);
+	if (op_errno)
+		goto error;
+	/*
+	 * check for cached info 
+	 */
+	op_ret = inode_ctx_get(fd->inode, this, &value);
+	if (op_ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log (this->name, GF_LOG_WARNING,
+				"Inode info was not found");
+			op_errno = EINVAL;
+			goto error;
+		}
+		/*
+		 * info was found in the cache
+		 */
+		local->info = info;
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info wasn't found in the cache;
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info)
+			goto error;
+		init_inode_info_head(info, fd);
+		local->info = info;
+		upload_info = _gf_true;
+	}
+	op_errno = open_format((unsigned char *)old_mtd->data,
+			       old_mtd->len,
+			       local->loc,
+			       info,
+			       get_master_cinfo(priv),
+			       local,
+			       upload_info);
+	if (op_errno)
+		goto error;
+	if (upload_info == _gf_true) {
+		op_errno = init_inode_info_tail(info,
+						get_master_cinfo(priv));
+		if (op_errno)
+			goto error;
+		op_errno = inode_ctx_put(fd->inode, this,
+					 (uint64_t)(long)(info));
+		if (op_errno == -1) {
+			op_errno = EIO;
+			goto error;
+		}
+	}
+	/*
+	 * update the format string (append/update/cup a MAC)
+	 */
+	op_errno = update_format(local->format,
+				 (unsigned char *)old_mtd->data,
+				 old_mtd->len,
+				 local->mac_idx,
+				 mop,
+				 local->newloc,
+				 info,
+				 get_master_cinfo(priv),
+				 local);
+	if (op_errno)
+		goto error;
+	/*
+	 * store the new format string on the server
+	 */
+	if (new_mtd_size) {
+		op_errno = dict_set_static_bin(local->xattr,
+					       CRYPTO_FORMAT_PREFIX,
+					       local->format,
+					       new_mtd_size);
+		if (op_errno)
+			goto error;
+	}
+	STACK_WIND(frame,
+		   do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   local->loc,
+		   local->xattr,
+		   0,
+		   NULL);
+	return 0;
+ wind:
+	wind_fn(frame, this);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t linkop_grab_local(call_frame_t *frame,
+				 xlator_t *this,
+				 loc_t *oldloc,
+				 loc_t *newloc,
+				 int flags, dict_t *xdata,
+				 glusterfs_fop_t op)
+{
+	int32_t ret = ENOMEM;
+	fd_t *fd;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, op);
+	if (!local)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	fd = fd_create(oldloc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->flags = flags;
+	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, oldloc);
+	if (ret) {
+		GF_FREE(local->loc);
+		local->loc = NULL;
+		goto error;
+	}
+	if (newloc) {
+		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
+		if (!local->newloc) {
+			loc_wipe(local->loc);
+			GF_FREE(local->loc);
+			goto error;
+		}
+		memset(local->newloc, 0, sizeof(*local->newloc));
+		ret = loc_copy(local->newloc, newloc);
+		if (ret) {
+			loc_wipe(local->loc);
+			GF_FREE(local->loc);
+			GF_FREE(local->newloc);
+			goto error;
+		}
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	return 0;
+
+error:
+        if (local) {
+                if (local->xdata)
+                        dict_unref(local->xdata);
+                if (local->fd)
+                        fd_unref(local->fd);
+                local->fd = 0;
+                local->loc = NULL;
+                local->newloc = NULL;
+                local->op_ret = -1;
+                local->op_errno = ret;
+        }
+
+        return ret;
+}
+
+/*
+ * read and verify locked metadata against the old pathname (via open);
+ * update the metadata string in accordance with the new pathname;
+ * submit modified metadata;
+ * wind;
+ */
+static int32_t linkop(call_frame_t *frame,
+		      xlator_t *this,
+		      loc_t *oldloc,
+		      loc_t *newloc,
+		      int flags,
+		      dict_t *xdata,
+		      glusterfs_fop_t op)
+{
+	int32_t ret;
+	dict_t *dict;
+	crypt_local_t *local;
+	void (*unwind_fn)(call_frame_t *frame);
+
+	unwind_fn = linkop_unwind_dispatch(op);
+
+	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
+	local = frame->local;
+	if (ret)
+		goto error;
+	dict = dict_new();
+	if (!dict) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * Set a message to crypt_open() that we need
+	 * locked metadata string.
+	 * All link operations (link, unlink, rename)
+	 * need write lock
+	 */
+	msgflags_set_mtd_wlock(&local->msgflags);
+	ret = dict_set_static_bin(dict,
+				  MSGFLAGS_PREFIX,
+				  &local->msgflags,
+				  sizeof(local->msgflags));
+	if (ret) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
+		dict_unref(dict);
+		goto error;
+	}
+	/*
+	 * verify metadata against the old pathname
+	 * and retrieve locked metadata string
+	 */
+	STACK_WIND(frame,
+		   linkop_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   oldloc,
+		   O_RDWR,
+		   local->fd,
+		   dict);
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = ret;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
+			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
+}
+
+static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
+			    loc_t *loc, int flags, dict_t *xdata)
+{
+	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
+}
+
+static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
+			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
+}
+
+static void put_one_call_open(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		fd_t *fd = local->fd;
+		loc_t *loc = local->loc;
+		dict_t *xdata = local->xdata;
+
+		STACK_UNWIND_STRICT(open,
+				    frame,
+				    local->op_ret,
+				    local->op_errno,
+				    fd,
+				    xdata);
+		fd_unref(fd);
+		if (xdata)
+			dict_unref(xdata);
+		loc_wipe(loc);
+		GF_FREE(loc);
+	}
+}
+
+static int32_t __crypt_readv_done(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	/* read deals with data configs only */
+	struct iovec *avec = local->data_conf.avec;
+	char **pool = local->data_conf.pool;
+	int blocks_in_pool = local->data_conf.blocks_in_pool;
+	struct iobref *iobref = local->iobref;
+	struct iobref *iobref_data = local->iobref_data;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "readv unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	dump_plain_text(local, avec);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
+	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
+	       (int)(local->rw_count > 0 ? iovec_get_size(avec, local->data_conf.acount) : 0),
+	       (unsigned long long)local->buf.ia_size);
+
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    local->rw_count > 0 ? local->rw_count : local->op_ret,
+			    local->op_errno,
+			    avec,
+			    avec ? local->data_conf.acount : 0,
+			    &local->buf,
+			    local->iobref,
+			    local_xdata);
+
+	free_avec(avec, pool, blocks_in_pool);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobref)
+		iobref_unref(iobref);
+	if (iobref_data)
+		iobref_unref(iobref_data);
+	return 0;
+}
+
+static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
+{
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 *  don't unlock (it will be done by the parent)
+		 */
+		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		crypt_local_t *local = frame->local;
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_readv_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+}
+
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local))
+		crypt_readv_done(frame, this);
+}
+
+static int32_t __crypt_writev_done(call_frame_t *frame,
+				   void *cookie,
+				   xlator_t *this,
+				   int32_t op_ret,
+				   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	int32_t ret_to_user;
+
+	if (local->xattr)
+		dict_unref(local->xattr);
+	/*
+	 * Calculate amout of butes to be returned
+	 * to user. We need to subtract paddings that
+	 * have been written as a part of atom.
+	 */
+	/*
+	 * subtract head padding
+	 */
+	if (local->rw_count == 0)
+		/*
+		 * Nothing has been written, it must be an error
+		 */
+		ret_to_user = local->op_ret;
+	else if (local->rw_count <= local->data_conf.off_in_head) {
+		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
+		ret_to_user = 0;
+	}
+	else
+		ret_to_user = local->rw_count -
+			local->data_conf.off_in_head;
+	/*
+	 * subtract tail padding
+	 */
+	if (ret_to_user > local->data_conf.orig_size)
+		ret_to_user = local->data_conf.orig_size;
+
+	if (local->iobref)
+		iobref_unref(local->iobref);
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "writev: ret_to_user: %d", ret_to_user);
+
+	STACK_UNWIND_STRICT(writev,
+			    frame,
+			    ret_to_user,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_writev_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 * don't unlock (it will be done by the parent)
+		 */
+		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_writev_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+	return 0;
+}
+
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_writev_done(frame, NULL,
+						  this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_writev_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+static int32_t __crypt_ftruncate_done(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	char *iobase = local->vec.iov_base;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "ftruncate unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "ftruncate, return to user: presize=%llu, postsize=%llu",
+	       (unsigned long long)local->prebuf.ia_size,
+	       (unsigned long long)local->postbuf.ia_size);
+
+	STACK_UNWIND_STRICT(ftruncate,
+			    frame,
+			    local->op_ret < 0 ? -1 : 0,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobase)
+		GF_FREE(iobase);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_done(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct gf_flock  lock  = {0, };
+
+	dict_unref(local->xattr);
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	lock.l_type   = F_UNLCK;
+	lock.l_whence = SEEK_SET;
+	lock.l_start  = 0;
+	lock.l_len    = 0;
+	lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __crypt_ftruncate_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+}
+
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_ftruncate_done(frame, NULL,
+						     this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_ftruncate_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+/*
+ * load regular file size for some FOPs
+ */
+static int32_t load_file_size(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      dict_t *dict,
+			      dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0)
+		goto unwind;
+	/*
+	 * load regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		if (local->xdata)
+			dict_unref(local->xdata);
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto unwind;
+	}
+	local->buf.ia_size = data_to_uint64(data);
+
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "FOP %d: Translate regular file to %llu",
+	       local->fop,
+	       (unsigned long long)local->buf.ia_size);
+ unwind:	
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_LOOKUP:
+		STACK_UNWIND_STRICT(lookup,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? local->inode : NULL,
+				    op_ret >= 0 ? &local->buf : NULL,				    
+				    local->xdata,
+				    op_ret >= 0 ? &local->postbuf : NULL);
+		break;
+	case GF_FOP_READ:
+		STACK_UNWIND_STRICT(readv,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    NULL,
+				    0,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    NULL,
+				    NULL);
+		break;
+	default:
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Improper file operation %d", local->fop);
+	}
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (local_inode)
+		inode_unref(local_inode);
+	return 0;
+}
+
+static int32_t crypt_stat_common_cbk(call_frame_t *frame,
+				     void *cookie,
+				     xlator_t *this,
+				     int32_t op_ret,
+				     int32_t op_errno,
+				     struct iatt *buf, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->buf = *buf;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->fgetxattr,
+			   local->fd,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->getxattr,
+			   local->loc,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+ unwind:
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+}
+
+static int32_t crypt_fstat(call_frame_t *frame,
+			   xlator_t *this,
+			   fd_t *fd, dict_t *xdata)
+{
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
+	if (!local)
+		goto error;
+	local->fd = fd_ref(fd);
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(fstat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_stat(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->stat,
+		   loc,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(stat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_lookup_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				inode_t *inode,
+				struct iatt *buf, dict_t *xdata,
+				struct iatt *postparent)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->postbuf = *postparent;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	uuid_copy(local->loc->gfid, buf->ia_gfid);
+
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ unwind:
+	loc_wipe(local->loc);
+	GF_FREE(local->loc);
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    inode,
+			    buf,
+			    xdata,
+			    postparent);
+	return 0;
+}
+
+static int32_t crypt_lookup(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
+	STACK_WIND(frame,
+		   crypt_lookup_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->lookup,
+		   loc,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * for every regular directory entry find its real file size
+ * and update stat's buf properly
+ */
+static int32_t crypt_readdirp_cbk(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno,
+				  gf_dirent_t *entries, dict_t *xdata)
+{
+	gf_dirent_t *entry = NULL;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	list_for_each_entry (entry, (&entries->list), list) {
+		data_t *data;
+
+		if (!IA_ISREG(entry->d_stat.ia_type))
+			continue;
+		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
+		if (!data){
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size of direntry not found");
+			op_errno = EIO;
+			op_ret = -1;
+			break;
+		}
+		entry->d_stat.ia_size = data_to_uint64(data);
+	}
+ unwind:
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+/*
+ * ->readdirp() fills in-core inodes, so we need to set proper
+ * file sizes for all directory entries of the parent @fd.
+ * Actual updates take place in ->crypt_readdirp_cbk()
+ */
+static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
+			      fd_t *fd, size_t size, off_t offset,
+			      dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+
+	if (!xdata) {
+		xdata = dict_new();
+		if (!xdata)
+			goto error;
+	}
+	else
+		dict_ref(xdata);
+	/*
+	 * make sure that we'll have real file sizes at ->readdirp_cbk()
+	 */
+	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(xdata);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_readdirp_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp,
+		   fd,
+		   size,
+		   offset,
+		   xdata);
+	dict_unref(xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_access(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t mask, dict_t *xdata)
+{
+	gf_log(this->name, GF_LOG_WARNING,
+	       "NFS mounts of encrypted volumes are unsupported");
+	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
+	return 0;
+}
+
+int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
+			       dict_t *options)
+{
+	uint64_t block_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("block-size", block_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("block-size", block_size, size, error);
+
+	switch (block_size) {
+	case 512:
+		master->m_block_bits = 9;
+		break;
+	case 1024:
+		master->m_block_bits = 10;
+		break;
+	case 2048:
+		master->m_block_bits = 11;
+		break;
+	case 4096:
+		master->m_block_bits = 12;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: unsupported block size %llu",
+		       (unsigned long long)block_size);
+		goto error;
+	}
+	return 0;
+ error:
+	return -1;
+}
+
+int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_alg = AES_CIPHER_ALG;
+	return 0;
+}
+
+int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_mode = XTS_CIPHER_MODE;
+	return 0;
+}
+
+/*
+ * set key size in bits to the master info
+ * Pre-conditions: cipher mode in the master info is uptodate.
+ */
+static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
+				     dict_t *options)
+{
+	int32_t ret;
+	uint64_t key_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("data-key-size", key_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("data-key-size", key_size, size, error);
+
+	ret = data_cipher_algs[master->m_alg][master->m_mode].check_key(key_size);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: wrong bin key size %llu for alg %d mode %d",
+		       (unsigned long long)key_size,
+		       (int)master->m_alg,
+		       (int)master->m_mode);
+		goto error;
+	}
+	master->m_dkey_size = key_size;
+	return 0;
+ error:
+	return -1;
+}
+
+static int is_hex(char *s) {
+	return  ('0' <= *s && *s <= '9') || ('a' <= *s && *s <= 'f');
+}
+
+static int parse_hex_buf(xlator_t *this, char *src, unsigned char *dst,
+			 int hex_size)
+{
+	int i;
+	int hex_byte = 0;
+
+	for (i = 0; i < (hex_size / 2); i++) {
+		if (!is_hex(src + i*2) || !is_hex(src + i*2 + 1)) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: not hex symbol in key");
+			return -1;
+		}
+		if (sscanf(src + i*2, "%2x", &hex_byte) != 1) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: can not parse hex key");
+			return -1;
+		}
+		dst[i] = hex_byte & 0xff;
+	}
+	return 0;
+}
+
+/*
+ * Parse options;
+ * install master volume key
+ */
+int32_t master_set_master_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	int32_t ret;
+	FILE *file = NULL;
+
+	int32_t key_size;
+	char *opt_key_file_pathname = NULL;
+
+	unsigned char bin_buf[MASTER_VOL_KEY_SIZE];
+	char hex_buf[2 * MASTER_VOL_KEY_SIZE];
+
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	/*
+	 * extract master key passed via option
+	 */
+	GF_OPTION_INIT("master-key", opt_key_file_pathname, path, bad_key);
+
+	if (!opt_key_file_pathname) {
+		gf_log(this->name, GF_LOG_ERROR, "FATAL: missing master key");
+		return -1;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "handling file key %s",
+	       opt_key_file_pathname);
+
+	file = fopen(opt_key_file_pathname, "r");
+	if (file == NULL) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: can not open file with master key");
+		return -1;
+	}
+	/*
+	 * extract hex key
+	 */
+	key_size = fread(hex_buf, 1, sizeof(hex_buf), file);
+	if (key_size < sizeof(hex_buf)) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: master key is too short");
+		goto bad_key;
+	}
+	ret = parse_hex_buf(this, hex_buf, bin_buf, key_size);
+	if (ret)
+		goto bad_key;
+	memcpy(master->m_key, bin_buf, MASTER_VOL_KEY_SIZE);
+	memset(hex_buf, 0, sizeof(hex_buf));
+	fclose(file);
+
+ 	memset(bin_buf, 0, sizeof(bin_buf));
+	return 0;
+ bad_key:
+	gf_log(this->name, GF_LOG_ERROR, "FATAL: bad master key");
+	if (file)
+		fclose(file);
+	memset(bin_buf, 0, sizeof(bin_buf));
+	return -1;
+}
+
+/*
+ * Derive volume key for object-id authentication
+ */
+int32_t master_set_nmtd_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	return get_nmtd_vol_key(get_master_cinfo(priv));
+}
+
+int32_t crypt_init_xlator(xlator_t *this)
+{
+	int32_t ret;
+	crypt_private_t *priv = this->private; 
+
+	ret = master_set_alg(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_mode(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_block_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_data_key_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_master_vol_key(this, priv);
+	if (ret)
+		return ret;
+	return master_set_nmtd_vol_key(this, priv);
+}
+
+static int32_t crypt_alloc_private(xlator_t *this)
+{
+	this->private = GF_CALLOC(1, sizeof(crypt_private_t), gf_crypt_mt_priv);
+	if (!this->private) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Can not allocate memory for private data");
+		return ENOMEM;
+	}
+	return 0;
+}
+
+static void crypt_free_private(xlator_t *this)
+{
+	crypt_private_t *priv = this->private;
+	if (priv) {
+		memset(priv, 0, sizeof(*priv));
+		GF_FREE(priv);
+	}
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_crypt_mt_end);
+
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                       "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t reconfigure (xlator_t *this, dict_t *options)
+{
+	int32_t ret = -1;
+	crypt_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO ("crypt", this, error);
+	GF_VALIDATE_OR_GOTO (this->name, this->private, error);
+	GF_VALIDATE_OR_GOTO (this->name, options, error);
+
+	priv = this->private;
+
+	ret = master_set_block_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure block size");
+		goto error;
+	}
+	ret = master_set_data_key_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure data key size");
+		goto error;
+	}
+	return 0;
+ error:
+	return ret;
+}
+
+int32_t init(xlator_t *this)
+{
+	int32_t ret;
+
+	if (!this->children || this->children->next) {
+		gf_log ("crypt", GF_LOG_ERROR,
+			"FATAL: crypt should have exactly one child");
+		return EINVAL;
+	}
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+	ret = crypt_alloc_private(this);
+	if (ret)
+		return ret;
+	ret = crypt_init_xlator(this);
+	if (ret)
+		goto error;
+	this->local_pool = mem_pool_new(crypt_local_t, 64);
+        if (!this->local_pool) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "failed to create local_t's memory pool");
+		ret = ENOMEM;
+                goto error;
+        }
+	gf_log ("crypt", GF_LOG_INFO, "crypt xlator loaded");
+	return 0;
+ error:
+	crypt_free_private(this);
+	return ret;
+}
+
+void fini (xlator_t *this)
+{
+	crypt_free_private(this);
+}
+
+struct xlator_fops fops = {
+	.readv        = crypt_readv,
+	.writev       = crypt_writev,
+	.truncate     = crypt_truncate,
+	.ftruncate    = crypt_ftruncate,
+	.setxattr     = crypt_setxattr,
+	.fsetxattr    = crypt_fsetxattr,
+	.link         = crypt_link,
+	.unlink       = crypt_unlink,
+	.rename       = crypt_rename,
+	.open         = crypt_open,
+	.create       = crypt_create,
+	.stat         = crypt_stat,
+	.fstat        = crypt_fstat,
+	.lookup       = crypt_lookup,
+	.readdirp     = crypt_readdirp,
+	.access       = crypt_access   
+};
+
+struct xlator_cbks cbks = {
+	.forget       = crypt_forget
+};
+
+struct volume_options options[] = {
+	{ .key = {"master-key"},
+	  .type = GF_OPTION_TYPE_PATH,
+	  .description = "Pathname of regular file which contains master volume key"
+	},
+	{ .key = {"data-key-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Data key size (bits)",
+	  .min = 256,
+	  .max = 512,
+	  .default_value = "256",
+	},
+	{ .key = {"block-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Atom size (bits)",
+	  .min = 512,
+	  .max = 4096,
+	  .default_value = "4096"
+	},
+	{ .key  = {NULL} },
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h
new file mode 100644
index 000000000..ff8eb571b
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.h
@@ -0,0 +1,903 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_H__
+#define __CRYPT_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <openssl/aes.h>
+#include <openssl/evp.h>
+#include <openssl/sha.h>
+#include <openssl/hmac.h>
+#include <openssl/cmac.h>
+#include <openssl/modes.h>
+#include "crypt-mem-types.h"
+
+#define CRYPT_XLATOR_ID  (0)
+
+#define MAX_IOVEC_BITS (3)
+#define MAX_IOVEC (1 << MAX_IOVEC_BITS)
+#define KEY_FACTOR_BITS  (6)
+
+#define DEBUG_CRYPT (0)
+#define TRIVIAL_TFM (0)
+
+#define CRYPT_MIN_BLOCK_BITS (9)
+#define CRYPT_MAX_BLOCK_BITS (12)
+
+#define MASTER_VOL_KEY_SIZE (32)
+#define NMTD_VOL_KEY_SIZE (16)
+
+#ifdef __NetBSD__
+typedef off_t loff_t;
+#endif
+
+struct crypt_key {
+	uint32_t len;
+	const char *label;	
+};
+
+/*
+ * Add new key types to the end of this
+ * enumeration but before LAST_KEY_TYPE
+ */
+typedef enum {
+	MASTER_VOL_KEY,
+	NMTD_VOL_KEY,
+	NMTD_LINK_KEY,
+	EMTD_FILE_KEY,
+	DATA_FILE_KEY_256,
+	DATA_FILE_KEY_512,
+	LAST_KEY_TYPE
+}crypt_key_type;
+
+struct kderive_context {
+	const unsigned char *pkey;/* parent key */
+	uint32_t pkey_len;        /* parent key size, bits */
+	uint32_t ckey_len;        /* child key size, bits */
+	unsigned char *fid;       /* fixed input data, NIST 800-108, 5.1 */
+	uint32_t fid_len;         /* fid len, bytes */
+	unsigned char *out;       /* contains child keying material */
+	uint32_t out_len;         /* out len, bytes */
+};
+
+typedef enum {
+	DATA_ATOM,
+	HOLE_ATOM,
+	LAST_DATA_TYPE
+}atom_data_type;
+
+typedef enum {
+	HEAD_ATOM,
+	TAIL_ATOM,
+	FULL_ATOM,
+	LAST_LOCALITY_TYPE
+}atom_locality_type;
+
+typedef enum {
+	MTD_CREATE,
+	MTD_APPEND,
+	MTD_OVERWRITE,
+	MTD_CUT,
+	MTD_LAST_OP
+} mtd_op_t;
+
+struct xts128_context {
+	void      *key1, *key2;
+	block128_f block1,block2;
+};
+
+struct object_cipher_info {
+	cipher_alg_t  o_alg;
+        cipher_mode_t o_mode;
+	uint32_t      o_block_bits;
+        uint32_t      o_dkey_size; /* raw data key size in bits */
+	union {
+		struct {
+			unsigned char ivec[16];
+			AES_KEY dkey[2];
+			AES_KEY tkey;    /* key used for tweaking */
+			XTS128_CONTEXT xts;
+		} aes_xts;
+	} u;
+};
+
+struct master_cipher_info {
+	/*
+	 * attributes inherited by newly created regular files
+	 */
+	cipher_alg_t  m_alg;
+        cipher_mode_t m_mode;
+	uint32_t      m_block_bits;
+        uint32_t      m_dkey_size; /* raw key size in bits */
+	/*
+	 * master key
+	 */
+	unsigned char m_key[MASTER_VOL_KEY_SIZE];
+	/* 
+	 * volume key for oid authentication
+	 */
+	unsigned char m_nmtd_key[NMTD_VOL_KEY_SIZE];
+};
+
+/*
+* This info is not changed during file's life
+ */
+struct crypt_inode_info {
+#if DEBUG_CRYPT
+	loc_t *loc; /* pathname that the file has been
+		       opened, or created with */
+#endif
+	uint16_t nr_minor;
+	uuid_t oid;
+	struct object_cipher_info cinfo;
+};
+
+/*
+ * this should locate in secure memory
+ */
+typedef struct {
+	struct master_cipher_info master;
+} crypt_private_t;
+
+static inline struct master_cipher_info *get_master_cinfo(crypt_private_t *priv)
+{
+	return &priv->master;
+}
+
+static inline struct object_cipher_info *get_object_cinfo(struct crypt_inode_info
+							 *info)
+{
+	return &info->cinfo;
+}
+
+/*
+ * this describes layouts and properties
+ * of atoms in an aligned vector
+ */
+struct avec_config {
+	uint32_t atom_size;
+	atom_data_type type;
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head;
+	uint32_t off_in_tail;
+	uint32_t gap_in_tail;
+	uint32_t nr_full_blocks;
+
+	struct iovec *avec; /* aligned vector */
+	uint32_t acount; /* number of avec components. The same
+			  * as number of occupied logical blocks */
+	char **pool;
+	uint32_t blocks_in_pool;
+	uint32_t cursor; /* makes sense only for ordered writes,
+			  * so there is no races on this counter.
+			  *
+			  * Cursor is per-config object, we don't
+			  * reset cursor for atoms of different
+			  * localities (head, tail, full)
+			  */
+};
+
+
+typedef struct {
+	glusterfs_fop_t fop; /* code of FOP this local info built for */
+	fd_t *fd;
+	inode_t *inode;
+	loc_t *loc;
+	int32_t mac_idx;
+	loc_t *newloc;
+	int32_t flags;
+	int32_t wbflags;
+	struct crypt_inode_info *info;
+	struct iobref *iobref;
+	struct iobref *iobref_data;
+	off_t offset;
+
+	uint64_t old_file_size; /* per FOP, retrieved under lock held */
+	uint64_t cur_file_size; /* per iteration, before issuing IOs */
+	uint64_t new_file_size; /* per iteration, after issuing IOs */
+
+	uint64_t io_offset; /* offset of IOs issued per iteration */
+	uint64_t io_offset_nopad; /* offset of user's data in the atom */
+	uint32_t io_size; /* size of IOs issued per iteration */
+	uint32_t io_size_nopad; /* size of user's data in the IOs */
+	uint32_t eof_padding_size; /* size od EOF padding in the IOs */
+
+	gf_lock_t call_lock; /* protect nr_calls from many cbks */
+	int32_t nr_calls;
+
+	atom_data_type active_setup; /* which setup (hole or date)
+					is currently active */
+	/* data setup */
+	struct avec_config data_conf;
+
+	/* hole setup */
+	int hole_conv_in_proggress;
+	gf_lock_t hole_lock; /* protect hole config from many cbks */
+	int hole_handled;
+	struct avec_config hole_conf;
+	struct iatt buf;
+	struct iatt prebuf;
+	struct iatt postbuf;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+	int32_t op_ret;
+	int32_t op_errno;
+	int32_t rw_count; /* total read or written */
+	gf_lock_t rw_count_lock; /* protect the counter above */
+	unsigned char *format; /* for create, update format string */
+	uint32_t format_size;
+	uint32_t msgflags; /* messages for crypt_open() */
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iovec vec; /* contains last file's atom for
+			     read-prune-write sequence */
+	gf_boolean_t custom_mtd;
+	/*
+	 * the next 3 fields are used by readdir and friends
+	 */
+	gf_dirent_t *de; /* directory entry */
+	char *de_path; /* pathname of directory entry */
+	uint32_t de_prefix_len; /* lenght of the parent's pathname */
+	gf_dirent_t *entries;
+
+	uint32_t update_disk_file_size:1;
+} crypt_local_t;
+
+/* This represents a (read)modify-write atom */
+struct rmw_atom {
+	atom_locality_type locality;
+	/*
+	 * read-modify-write sequence of the atom
+	 */
+	int32_t (*rmw)(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iovec *vec,
+		       int32_t count,
+		       struct iatt *stbuf,
+		       struct iobref *iobref,
+		       dict_t *xdata);
+	/*
+	 * offset of the logical block in a file
+	 */
+	loff_t (*offset_at)(call_frame_t *frame,
+			    struct object_cipher_info *object);
+	/*
+	 * IO offset in an atom
+	 */
+	uint32_t (*offset_in)(call_frame_t *frame,
+			      struct object_cipher_info *object);
+	/*
+	 * number of bytes of plain text of this atom that user
+	 * wants to read/write.
+	 * It can be smaller than atom_size in the case of head
+	 * or tail atoms.
+	 */
+	uint32_t (*io_size_nopad)(call_frame_t *frame,
+				  struct object_cipher_info *object);
+	/*
+	 * which iovec represents the atom
+	 */
+	struct iovec *(*get_iovec)(call_frame_t *frame, uint32_t count);
+	/*
+	 * how many bytes of partial block should be uptodated by
+	 * reading from disk.
+	 * This is used to perform a read component of RMW (read-modify-write).
+	 */
+	uint32_t (*count_to_uptodate)(call_frame_t *frame, struct object_cipher_info *object);
+	struct avec_config *(*get_config)(call_frame_t *frame);
+};
+
+struct data_cipher_alg {
+	gf_boolean_t atomic; /* true means that algorithm requires
+				to pad data before cipher transform */
+	gf_boolean_t should_pad; /* true means that algorithm requires
+				    to pad the end of file with extra-data */
+	uint32_t blkbits; /* blksize = 1 << blkbits */
+	/*
+	 * any preliminary sanity checks goes here
+	 */
+	int32_t (*init)(void);
+	/*
+	 * set alg-mode specific inode info
+	 */
+	int32_t (*set_private)(struct crypt_inode_info *info,
+			       struct master_cipher_info *master);
+	/*
+	 * check alg-mode specific data key
+	 */
+	int32_t (*check_key)(uint32_t key_size);
+	void (*set_iv)(off_t offset, struct object_cipher_info *object);
+	int32_t (*encrypt)(const unsigned char *from, unsigned char *to,
+			   size_t length, off_t offset, const int enc,
+			   struct object_cipher_info *object);
+};
+
+/*
+ * version-dependent metadata loader
+ */
+struct crypt_mtd_loader {
+	/*
+	 * return core format size
+	 */
+	size_t (*format_size)(mtd_op_t op, size_t old_size);
+	/*
+	 * pack version-specific metadata of an object
+	 * at ->create()
+	 */
+	int32_t (*create_format)(unsigned char *wire,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master);
+	/*
+	 * extract version-specific metadata of an object
+	 * at ->open() time
+	 */
+	int32_t (*open_format)(unsigned char *wire,
+			       int32_t len,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local,
+			       gf_boolean_t load_info);
+	int32_t (*update_format)(unsigned char *new,
+				 unsigned char *old,
+				 size_t old_len,
+				 int32_t mac_idx,
+				 mtd_op_t op,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master,
+				 crypt_local_t *local);
+};
+
+typedef int32_t (*end_writeback_handler_t)(call_frame_t *frame,
+					   void *cookie,
+					   xlator_t *this,
+					   int32_t op_ret,
+					   int32_t op_errno,
+					   struct iatt *prebuf,
+					   struct iatt *postbuf,
+					   dict_t *xdata);
+typedef void (*linkop_wind_handler_t)(call_frame_t *frame, xlator_t *this);
+typedef void (*linkop_unwind_handler_t)(call_frame_t *frame);
+
+
+/* Declarations */
+
+/* keys.c */
+extern struct crypt_key crypt_keys[LAST_KEY_TYPE];
+int32_t get_nmtd_vol_key(struct master_cipher_info *master);
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key);
+/* data.c */
+extern struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE];
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf);
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count);
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop);
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype);
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t setup_gap_in_tail);
+
+/* metadata.c */
+extern struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER];
+
+int32_t alloc_format(crypt_local_t *local, size_t size);
+int32_t alloc_format_create(crypt_local_t *local);
+void free_format(crypt_local_t *local);
+size_t format_size(mtd_op_t op, size_t old_size);
+size_t new_format_size(void);
+int32_t open_format(unsigned char *str, int32_t len, loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master, crypt_local_t *local,
+		    gf_boolean_t load_info);
+int32_t update_format(unsigned char *new, unsigned char *old,
+		      size_t old_len, int32_t mac_idx, mtd_op_t op, loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local);
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master);
+
+/* atom.c */
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality);
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype);
+void submit_full(call_frame_t *frame, xlator_t *this);
+
+/* crypt.c */
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop);
+static size_t iovec_get_size(struct iovec *vec, uint32_t count);
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom, off_t io_offset,
+				uint32_t io_size);
+void link_wind(call_frame_t *frame, xlator_t *this);
+void unlink_wind(call_frame_t *frame, xlator_t *this);
+void link_unwind(call_frame_t *frame);
+void unlink_unwind(call_frame_t *frame);
+void rename_wind(call_frame_t *frame, xlator_t *this);
+void rename_unwind(call_frame_t *frame);
+
+/* Inline functions */
+
+static inline size_t iovec_get_size(struct iovec *vec, uint32_t count)
+{
+	int i;
+	size_t size = 0;
+	for (i = 0; i < count; i++)
+		size += vec[i].iov_len;
+	return size;
+}
+
+static inline int32_t crypt_xlator_id(void)
+{
+	return CRYPT_XLATOR_ID;
+}
+
+static inline mtd_loader_id current_mtd_loader(void)
+{
+	return MTD_LOADER_V1;
+}
+
+static inline uint32_t master_key_size (void)
+{
+	return crypt_keys[MASTER_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t nmtd_vol_key_size (void)
+{
+	return crypt_keys[NMTD_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t alg_mode_blkbits(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].blkbits;
+}
+
+static inline uint32_t alg_mode_blksize(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return 1 << alg_mode_blkbits(alg, mode);
+}
+
+static inline gf_boolean_t alg_mode_atomic(cipher_alg_t alg,
+					   cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].atomic;
+}
+
+static inline gf_boolean_t alg_mode_should_pad(cipher_alg_t alg,
+					       cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].should_pad;
+}
+
+static inline uint32_t master_alg_blksize(struct master_cipher_info *mr)
+{
+	return alg_mode_blksize(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t master_alg_blkbits(struct master_cipher_info *mr)
+{
+	return alg_mode_blkbits(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_atomic(struct master_cipher_info *mr)
+{
+	return alg_mode_atomic(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_should_pad(struct master_cipher_info *mr)
+{
+	return alg_mode_should_pad(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t object_alg_blksize(struct object_cipher_info *ob)
+{
+	return alg_mode_blksize(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t object_alg_blkbits(struct object_cipher_info *ob)
+{
+	return alg_mode_blkbits(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_atomic(struct object_cipher_info *ob)
+{
+	return alg_mode_atomic(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_should_pad(struct object_cipher_info *ob)
+{
+	return alg_mode_should_pad(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t aes_raw_key_size(struct master_cipher_info *master)
+{
+	return master->m_dkey_size >> 3;
+}
+
+static inline struct avec_config *get_hole_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->hole_conf);
+}
+
+static inline struct avec_config *get_data_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->data_conf);
+}
+
+static inline int32_t get_atom_bits (struct object_cipher_info *object)
+{
+	return object->o_block_bits;
+}
+
+static inline int32_t get_atom_size (struct object_cipher_info *object)
+{
+	return 1 << get_atom_bits(object);
+}
+
+static inline int32_t has_head_block(struct avec_config *conf)
+{
+	return conf->off_in_head ||
+		(conf->acount == 1 && conf->off_in_tail);
+}
+
+static inline int32_t has_tail_block(struct avec_config *conf)
+{
+	return conf->off_in_tail && conf->acount > 1;
+}
+
+static inline int32_t has_full_blocks(struct avec_config *conf)
+{
+	return conf->nr_full_blocks;
+}
+
+static inline int32_t should_submit_head_block(struct avec_config *conf)
+{
+	return has_head_block(conf) && (conf->cursor == 0);
+}
+
+static inline int32_t should_submit_tail_block(struct avec_config *conf)
+{
+	return has_tail_block(conf) && (conf->cursor == conf->acount - 1);
+}
+
+static inline int32_t should_submit_full_block(struct avec_config *conf)
+{
+	uint32_t start = has_head_block(conf) ? 1 : 0;
+
+	return has_full_blocks(conf) &&
+		conf->cursor >= start &&
+		conf->cursor < start + conf->nr_full_blocks;
+}
+
+#if DEBUG_CRYPT
+static inline void crypt_check_input_len(size_t len,
+					 struct object_cipher_info *object)
+{
+	if (object_alg_should_pad(object) && (len & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG, "bad input len: %d", (int)len);
+}
+
+static inline void check_head_block(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a head atom");
+}
+
+static inline void check_tail_block(struct avec_config *conf)
+{
+	if (!has_tail_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a tail atom");
+}
+
+static inline void check_full_block(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a full atom");
+}
+
+static inline void check_cursor_head(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of head atom method");
+	else if (conf->cursor != 0)
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor (%d) is not at head atom",
+		       conf->cursor);
+}
+
+static inline void check_cursor_full(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of full atom method");
+	if (has_head_block(conf) && (conf->cursor == 0))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor is not at full atom");
+}
+
+/*
+ * FIXME: use avec->iov_len to check setup
+ */
+static inline int data_local_invariant(crypt_local_t *local)
+{
+	return 0;
+}
+
+#else
+#define crypt_check_input_len(len, object) noop
+#define check_head_block(conf) noop
+#define check_tail_block(conf) noop
+#define check_full_block(conf) noop
+#define check_cursor_head(conf) noop
+#define check_cursor_full(conf) noop
+
+#endif /* DEBUG_CRYPT */
+
+static inline struct avec_config *conf_by_type(call_frame_t *frame,
+					       atom_data_type dtype)
+{
+	struct avec_config *conf = NULL;
+
+	switch (dtype) {
+	case HOLE_ATOM:
+		conf = get_hole_conf(frame);
+		break;
+	case DATA_ATOM:
+		conf = get_data_conf(frame);
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom type");
+	}
+	return conf;
+}
+
+static inline uint32_t nr_calls_head(struct avec_config *conf)
+{
+	return has_head_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_tail(struct avec_config *conf)
+{
+	return has_tail_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_full(struct avec_config *conf)
+{
+	switch(conf->type) {
+	case HOLE_ATOM:
+		return has_full_blocks(conf);
+	case DATA_ATOM:
+		return has_full_blocks(conf) ?
+			logical_blocks_occupied(0,
+						conf->nr_full_blocks,
+						MAX_IOVEC_BITS)	: 0;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom data type");
+		return 0;
+	}
+}
+
+static inline uint32_t nr_calls(struct avec_config *conf)
+{
+	return nr_calls_head(conf) + nr_calls_tail(conf) + nr_calls_full(conf);
+}
+
+static inline uint32_t nr_calls_data(call_frame_t *frame)
+{
+	return nr_calls(get_data_conf(frame));
+}
+
+static inline uint32_t nr_calls_hole(call_frame_t *frame)
+{
+	return nr_calls(get_hole_conf(frame));
+}
+
+static inline void get_one_call_nolock(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	++local->nr_calls;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", 1);
+}
+
+static inline void get_one_call(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_one_call_nolock(frame);
+	UNLOCK(&local->call_lock);
+}
+
+static inline void get_nr_calls_nolock(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	local->nr_calls += nr;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", nr);
+}
+
+static inline void get_nr_calls(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_nr_calls_nolock(frame, nr);
+	UNLOCK(&local->call_lock);
+}
+
+static inline int put_one_call(crypt_local_t *local)
+{
+	uint32_t last = 0;
+
+	LOCK(&local->call_lock);
+	if (--local->nr_calls == 0)
+		last = 1;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "put %d calls", 1);
+
+	UNLOCK(&local->call_lock);
+	return last;
+}
+
+static inline int is_appended_write(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->orig_offset + conf->orig_size > local->old_file_size;
+}
+
+static inline int is_ordered_mode(call_frame_t *frame)
+{
+#if 0
+	crypt_local_t *local = frame->local;
+	return local->fop == GF_FOP_FTRUNCATE ||
+		(local->fop == GF_FOP_WRITE && is_appended_write(frame));
+#endif
+	return 1;
+}
+
+static inline int32_t hole_conv_completed(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+	return conf->cursor == conf->acount;
+}
+
+static inline int32_t data_write_in_progress(crypt_local_t *local)
+{
+	return local->active_setup == DATA_ATOM;
+}
+
+static inline int32_t parent_is_crypt_xlator(call_frame_t *frame,
+					     xlator_t *this)
+{
+	return frame->parent->this == this;
+}
+
+static inline linkop_wind_handler_t linkop_wind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_wind;
+	case GF_FOP_UNLINK:
+		return unlink_wind;
+	case GF_FOP_RENAME:
+		return rename_wind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;
+	}
+}
+
+static inline linkop_unwind_handler_t linkop_unwind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_unwind;
+	case GF_FOP_UNLINK:
+		return unlink_unwind;
+	case GF_FOP_RENAME:
+		return rename_unwind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;		
+	}	
+}
+
+static inline mtd_op_t linkop_mtdop_dispatch(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_LINK:
+		return MTD_APPEND;
+	case GF_FOP_UNLINK:
+		return MTD_CUT;
+	case GF_FOP_RENAME:
+		return MTD_OVERWRITE;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad link operation %d", fop);
+		return MTD_LAST_OP;
+	}
+}
+
+#endif /* __CRYPT_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/data.c b/xlators/encryption/crypt/src/data.c
new file mode 100644
index 000000000..762fa554a
--- /dev/null
+++ b/xlators/encryption/crypt/src/data.c
@@ -0,0 +1,769 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void set_iv_aes_xts(off_t offset, struct object_cipher_info *object)
+{
+	unsigned char *ivec;
+
+	ivec = object->u.aes_xts.ivec;
+
+	/* convert the tweak into a little-endian byte
+	 * array (IEEE P1619/D16, May 2007, section 5.1)
+	 */
+
+	*((uint64_t *)ivec) = htole64(offset);
+
+	/* ivec is padded with zeroes */
+}
+
+static int32_t aes_set_keys_common(unsigned char *raw_key, uint32_t key_size,
+				   AES_KEY *keys)
+{
+	int32_t ret;
+
+	ret = AES_set_encrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_ENCRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set encrypt key failed");
+		return ret;
+	}
+	ret = AES_set_decrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_DECRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set decrypt key failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * set private cipher info for xts mode
+ */
+static int32_t set_private_aes_xts(struct crypt_inode_info *info,
+				   struct master_cipher_info *master)
+{
+	int ret;
+	struct object_cipher_info *object = get_object_cinfo(info);
+	unsigned char *data_key;
+	uint32_t subkey_size;
+
+	/* init tweak value */
+	memset(object->u.aes_xts.ivec, 0, 16);
+
+	data_key = GF_CALLOC(1, object->o_dkey_size, gf_crypt_mt_key);
+	if (!data_key)
+		return ENOMEM;
+
+	/*
+	 * retrieve data keying meterial
+	 */
+	ret = get_data_file_key(info, master, object->o_dkey_size, data_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Failed to retrieve data key");
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * parse compound xts key
+	 */
+	subkey_size = object->o_dkey_size >> 4; /* (xts-key-size-in-bytes / 2) */
+	/*
+	 * install key for data encryption
+	 */
+	ret = aes_set_keys_common(data_key,
+				  subkey_size << 3, object->u.aes_xts.dkey);
+	if (ret) {
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * set up key used to encrypt tweaks
+	 */
+	ret = AES_set_encrypt_key(data_key + subkey_size,
+				  object->o_dkey_size / 2,
+				  &object->u.aes_xts.tkey);
+	if (ret < 0)
+		gf_log("crypt", GF_LOG_ERROR, "Set tweak key failed");
+
+	GF_FREE(data_key);
+	return ret;
+}
+
+static int32_t aes_xts_init(void)
+{
+	cassert(AES_BLOCK_SIZE == (1 << AES_BLOCK_BITS));
+	return 0;
+}
+
+static int32_t check_key_aes_xts(uint32_t keysize)
+{
+	switch(keysize) {
+	case 256:
+	case 512:
+		return 0;
+	default:
+		break;
+	}
+	return -1;
+}
+
+static int32_t encrypt_aes_xts(const unsigned char *from,
+			       unsigned char *to, size_t length,
+			       off_t offset, const int enc,
+			       struct object_cipher_info *object)
+{
+	XTS128_CONTEXT ctx;
+	if (enc) {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_ENCRYPT];
+		ctx.block1 = (block128_f)AES_encrypt;
+	}
+	else {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_DECRYPT];
+		ctx.block1 = (block128_f)AES_decrypt;
+	}
+	ctx.key2 = &object->u.aes_xts.tkey;
+	ctx.block2 = (block128_f)AES_encrypt;
+
+	return CRYPTO_xts128_encrypt(&ctx,
+				     object->u.aes_xts.ivec,
+				     from,
+				     to,
+				     length, enc);
+}
+
+/*
+ * Cipher input chunk @from of length @len;
+ * @to: result of cipher transform;
+ * @off: offset in a file (must be cblock-aligned);
+ */
+static void cipher_data(struct object_cipher_info *object,
+			char *from,
+			char *to,
+			off_t off,
+			size_t len,
+			const int enc)
+{
+	crypt_check_input_len(len, object);
+
+#if TRIVIAL_TFM && DEBUG_CRYPT
+	return;
+#endif
+	data_cipher_algs[object->o_alg][object->o_mode].set_iv(off, object);
+	data_cipher_algs[object->o_alg][object->o_mode].encrypt
+		((const unsigned char *)from,
+		 (unsigned char *)to,
+		 len,
+		 off,
+		 enc,
+		 object);
+}
+
+#define MAX_CIPHER_CHUNK (1 << 30)
+
+/*
+ * Do cipher (encryption/decryption) transform of a
+ * continuous region of memory.
+ *
+ * @len: a number of bytes to transform;
+ * @buf: data to transform;
+ * @off: offset in a file, should be block-aligned
+ *       for atomic cipher modes and ksize-aligned
+ *       for other modes).
+ * @dir: direction of transform (encrypt/decrypt).
+ */
+static void cipher_region(struct object_cipher_info *object,
+			  char *from,
+			  char *to,
+			  off_t off,
+			  size_t len,
+			  int dir)
+{
+	while (len > 0) {
+		size_t to_cipher;
+
+		to_cipher = len;
+		if (to_cipher > MAX_CIPHER_CHUNK)
+			to_cipher = MAX_CIPHER_CHUNK;
+
+		/* this will reset IV */
+		cipher_data(object,
+			    from,
+			    to,
+			    off,
+			    to_cipher,
+			    dir);
+		from += to_cipher;
+		to   += to_cipher;
+		off  += to_cipher;
+		len  -= to_cipher;
+	}
+}
+
+/*
+ * Do cipher transform (encryption/decryption) of
+ * plaintext/ciphertext represented by @vec.
+ *
+ * Pre-conditions: @vec represents a continuous piece
+ * of data in a file at offset @off to be ciphered
+ * (encrypted/decrypted).
+ * @count is the number of vec's components. All the
+ * components must be block-aligned, the caller is
+ * responsible for this. @dir is "direction" of
+ * transform (encrypt/decrypt).
+ */
+static void cipher_aligned_iov(struct object_cipher_info *object,
+			       struct iovec *vec,
+			       int count,
+			       off_t off,
+			       int32_t dir)
+{
+	int i;
+	int len = 0;
+
+	for (i = 0; i < count; i++) {
+		cipher_region(object,
+			      vec[i].iov_base,
+			      vec[i].iov_base,
+			      off + len,
+			      vec[i].iov_len,
+			      dir);
+		len += vec[i].iov_len;
+	}
+}
+
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 1);
+}
+
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 0);
+}
+
+#if DEBUG_CRYPT
+static void compound_stream(struct iovec *vec, int count, char *buf, off_t skip)
+{
+	int i;
+	int off = 0;
+	for (i = 0; i < count; i++) {
+		memcpy(buf + off,
+		       vec[i].iov_base + skip,
+		       vec[i].iov_len - skip);
+
+		off += (vec[i].iov_len - skip);
+		skip = 0;
+	}
+}
+
+static void check_iovecs(struct iovec *vec, int cnt,
+			 struct iovec *avec, int acnt, uint32_t off_in_head)
+{
+	char *s1, *s2;
+	uint32_t size, asize;
+
+	size = iovec_get_size(vec, cnt);
+	asize = iovec_get_size(avec, acnt) - off_in_head;
+	if (size != asize) {
+		gf_log("crypt", GF_LOG_DEBUG, "size %d is not eq asize %d",
+		       size, asize);
+		return;
+	}
+	s1 = GF_CALLOC(1, size, gf_crypt_mt_data);
+	if (!s1) {
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	s2 = GF_CALLOC(1, asize, gf_crypt_mt_data);
+	if (!s2) {
+		GF_FREE(s1);
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	compound_stream(vec, cnt, s1, 0);
+	compound_stream(avec, acnt, s2, off_in_head);
+	if (memcmp(s1, s2, size))
+		gf_log("crypt", GF_LOG_DEBUG, "chunks of different data");
+	GF_FREE(s1);
+	GF_FREE(s2);
+}
+
+#else
+#define check_iovecs(vec, count, avec, avecn, off) noop
+#endif /* DEBUG_CRYPT */
+
+static char *data_alloc_block(xlator_t *this, crypt_local_t *local,
+			      int32_t block_size)
+{
+	struct iobuf *iobuf = NULL;
+
+	iobuf = iobuf_get2(this->ctx->iobuf_pool, block_size);
+	if (!iobuf) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Failed to get iobuf");
+		return NULL;
+	}
+	if (!local->iobref_data) {
+		local->iobref_data = iobref_new();
+		if (!local->iobref_data) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "Failed to get iobref");
+			iobuf_unref(iobuf);
+			return NULL;
+		}
+	}
+	iobref_add(local->iobref_data, iobuf);
+	return iobuf->ptr;
+}
+
+/*
+ * Compound @avec, which represent the same data
+ * chunk as @vec, but has aligned components of
+ * specified block size. Alloc blocks, if needed.
+ * In particular, incomplete head and tail blocks
+ * must be allocated.
+ * Put number of allocated blocks to @num_blocks.
+ *
+ * Example:
+ *
+ * input: data chunk represented by 4 components
+ * [AB],[BC],[CD],[DE];
+ * output: 5 logical blocks (0, 1, 2, 3, 4).
+ *
+ *   A     B       C     D               E
+ *   *-----*+------*-+---*----+--------+-*
+ *   |     ||      | |   |    |        | |
+ * *-+-----+*------+-*---+----*--------*-+------*
+ * 0        1        2        3        4
+ *
+ * 0 - incomplete compound (head);
+ * 1, 2 - full compound;
+ * 3 - full non-compound (the case of reuse);
+ * 4 - incomplete non-compound (tail).
+ */
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf)
+{
+	int vecn = 0;      /* number of the current component in vec */
+	int avecn = 0;     /* number of the current component in avec */
+	off_t vec_off = 0; /* offset in the current vec component,
+			    * i.e. the number of bytes have already
+			    * been copied */
+	int32_t block_size = get_atom_size(object);
+	size_t to_process; /* number of vec's bytes to copy and(or) re-use */
+	int32_t off_in_head = conf->off_in_head;
+
+	to_process = iovec_get_size(vec, count);
+
+	while (to_process > 0) {
+		if (off_in_head ||
+		    vec[vecn].iov_len - vec_off < block_size) {
+			/*
+			 * less than block_size:
+			 * the case of incomplete (head or tail),
+			 * or compound block
+			 */
+			size_t copied = 0;
+			/*
+			 * populate the pool with a new block
+			 */
+			blocks[*blocks_allocated] = data_alloc_block(this,
+								    local,
+								    block_size);
+			if (!blocks[*blocks_allocated])
+				return -ENOMEM;
+			memset(blocks[*blocks_allocated], 0, off_in_head);
+			/*
+			 * fill the block with vec components
+			 */
+			do {
+				size_t to_copy;
+
+				to_copy = vec[vecn].iov_len - vec_off;
+				if (to_copy > block_size - off_in_head)
+					to_copy = block_size - off_in_head;
+
+				memcpy(blocks[*blocks_allocated] + off_in_head + copied,
+				       vec[vecn].iov_base + vec_off,
+				       to_copy);
+
+				copied += to_copy;
+				to_process -= to_copy;
+
+				vec_off += to_copy;
+				if (vec_off == vec[vecn].iov_len) {
+					/* finished with this vecn */
+					vec_off = 0;
+					vecn++;
+				}
+			} while (copied < (block_size - off_in_head) && to_process > 0);
+			/*
+			 * update avec
+			 */
+			avec[avecn].iov_len = off_in_head + copied;
+			avec[avecn].iov_base = blocks[*blocks_allocated];
+
+			(*blocks_allocated)++;
+			off_in_head = 0;
+		} else {
+			/*
+			 * the rest of the current vec component
+			 * is not less than block_size, so reuse
+			 * the memory buffer of the component.
+			 */
+			size_t to_reuse;
+			to_reuse = (to_process > block_size ?
+				    block_size :
+				    to_process);
+			avec[avecn].iov_len = to_reuse;
+			avec[avecn].iov_base = vec[vecn].iov_base + vec_off;
+
+			vec_off += to_reuse;
+			if (vec_off == vec[vecn].iov_len) {
+				/* finished with this vecn */
+				vec_off = 0;
+				vecn++;
+			}
+			to_process -= to_reuse;
+		}
+		avecn++;
+	}
+	check_iovecs(vec, count, avec, avecn, conf->off_in_head);
+	return 0;
+}
+
+/*
+ * allocate and setup aligned vector for data submission
+ * Pre-condition: @conf is set.
+ */
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count)
+{
+	int32_t ret = ENOMEM;
+	struct iovec *avec;
+	char **pool;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = DATA_ATOM;
+
+	avec = GF_CALLOC(conf->acount, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ret;
+	pool = GF_CALLOC(conf->acount, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ret;
+	}
+	if (!vec) {
+		/*
+		 * degenerated case: no data
+		 */
+		pool[0] = data_alloc_block(this, local, get_atom_size(object));
+		if (!pool[0])
+			goto free;
+		blocks_in_pool = 1;
+		avec->iov_base = pool[0];
+		avec->iov_len = conf->off_in_tail;
+	}
+	else {
+		ret = align_iov_by_atoms(this, local, object, vec, vec_count,
+					 avec, pool, &blocks_in_pool, conf);
+		if (ret)
+			goto free;
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ret;
+}
+
+/*
+ * allocate and setup aligned vector for hole submission
+ */
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop)
+{
+	uint32_t i, idx;
+	struct iovec *avec;
+	char **pool;
+	uint32_t num_blocks;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = HOLE_ATOM;
+
+	num_blocks = conf->acount -
+		(conf->nr_full_blocks ? conf->nr_full_blocks - 1 : 0);
+
+	switch (fop) {
+	case GF_FOP_WRITE:
+		/*
+		 * hole goes before data
+		 */
+		if (num_blocks == 1 && conf->off_in_tail != 0)
+		/*
+		 * we won't submit a hole which fits into
+		 * a data atom: this part of hole will be
+		 * submitted with data write
+		 */
+			return 0;
+		break;
+	case GF_FOP_FTRUNCATE:
+		/*
+		 * expanding truncate, hole goes after data,
+		 * and will be submited in any case.
+ 		 */
+		break;
+	default:
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad file operation %d", fop);
+		return 0;
+	}
+	avec = GF_CALLOC(num_blocks, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ENOMEM;
+	pool = GF_CALLOC(num_blocks, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ENOMEM;
+	}
+	for (i = 0; i < num_blocks; i++) {
+		pool[i] = data_alloc_block(this, local, get_atom_size(object));
+		if (pool[i] == NULL)
+			goto free;
+		blocks_in_pool++;
+	}
+	if (has_head_block(conf)) {
+		/* set head block */
+		idx = 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base + conf->off_in_head,
+		       0,
+		       get_atom_size(object) - conf->off_in_head);
+	}
+	if (has_tail_block(conf)) {
+		/* set tail block */
+		idx = num_blocks - 1;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base, 0, conf->off_in_tail);
+	}
+	if (has_full_blocks(conf)) {
+		/* set full block */
+		idx = conf->off_in_head ? 1 : 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		/*
+		 * since we re-use the buffer,
+		 * zeroes will be set every time
+		 * before encryption, see submit_full()
+		 */
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ENOMEM;
+}
+
+/* A helper for setting up config of partial atoms (which
+ * participate in read-modify-write sequence).
+ *
+ * Calculate and setup precise amount of "extra-bytes"
+ * that should be uptodated at the end of partial (not
+ * necessarily tail!) block.
+ *
+ * Pre-condition: local->old_file_size is valid!
+ * @conf contains setup, which is enough for correct calculation
+ * of has_tail_block(), ->get_offset().
+ */
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype)
+{
+	uint32_t to_block;
+	crypt_local_t *local = frame->local;
+	uint64_t old_file_size = local->old_file_size;
+	struct rmw_atom *partial = atom_by_types(dtype,
+						 has_tail_block(conf) ?
+						 TAIL_ATOM : HEAD_ATOM);
+
+	if (old_file_size <= partial->offset_at(frame, object))
+		to_block = 0;
+	else {
+		to_block = old_file_size - partial->offset_at(frame, object);
+		if (to_block > get_atom_size(object))
+			to_block = get_atom_size(object);
+	}
+	if (to_block > conf->off_in_tail)
+		conf->gap_in_tail = to_block - conf->off_in_tail;
+	else
+		/*
+		 * nothing to uptodate
+		 */
+		conf->gap_in_tail = 0;
+}
+
+/*
+ * fill struct avec_config with offsets layouts
+ */
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t set_gap)
+{
+	crypt_local_t *local;
+	struct object_cipher_info *object;
+	struct avec_config *conf;
+	uint32_t resid;
+
+	uint32_t atom_size;
+	uint32_t atom_bits;
+
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head = 0;
+	uint32_t off_in_tail = 0;
+	uint32_t nr_full_blocks;
+	int32_t size_full_blocks;
+
+	uint32_t acount; /* number of alifned components to write.
+			  * The same as number of occupied logical
+			  * blocks (atoms)
+			  */
+	local = frame->local;
+	object = &local->info->cinfo;
+	conf = (dtype == DATA_ATOM ?
+		get_data_conf(frame) : get_hole_conf(frame));
+
+	orig_offset = offset;
+	orig_size = count;
+
+	atom_size = get_atom_size(object);
+	atom_bits = get_atom_bits(object);
+
+	/*
+	 * Round-down the start,
+	 * round-up the end.
+	 */
+	resid = offset & (uint64_t)(atom_size - 1);
+
+	if (resid)
+		off_in_head = resid;
+	aligned_offset = offset - off_in_head;
+	expanded_size = orig_size + off_in_head;
+
+	/* calculate tail,
+	   expand size forward  */
+	resid = (offset + orig_size) & (uint64_t)(atom_size - 1);
+
+	if (resid) {
+		off_in_tail = resid;
+		expanded_size += (atom_size - off_in_tail);
+	}
+	/*
+	 * calculate number of occupied blocks
+	 */
+	acount = expanded_size >> atom_bits;
+	/*
+	 * calculate number of full blocks
+	 */
+	size_full_blocks = expanded_size;
+	if (off_in_head)
+		size_full_blocks -= atom_size;
+	if (off_in_tail && size_full_blocks > 0)
+		size_full_blocks -= atom_size;
+	nr_full_blocks = size_full_blocks >> atom_bits;
+
+	conf->atom_size = atom_size;
+	conf->orig_size = orig_size;
+	conf->orig_offset = orig_offset;
+	conf->expanded_size = expanded_size;
+	conf->aligned_offset = aligned_offset;
+
+	conf->off_in_head = off_in_head;
+	conf->off_in_tail = off_in_tail;
+	conf->nr_full_blocks = nr_full_blocks;
+	conf->acount = acount;
+	/*
+	 * Finally, calculate precise amount of
+	 * "extra-bytes" that should be uptodated
+	 * at the end.
+	 * Only if RMW is expected.
+	 */
+	if (off_in_tail && set_gap)
+		set_gap_at_end(frame, object, conf, dtype);
+}
+
+struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE] = {
+	[AES_CIPHER_ALG][XTS_CIPHER_MODE] =
+	{ .atomic = _gf_true,
+	  .should_pad = _gf_true,
+	  .blkbits = AES_BLOCK_BITS,
+	  .init = aes_xts_init,
+	  .set_private = set_private_aes_xts,
+	  .check_key = check_key_aes_xts,
+	  .set_iv = set_iv_aes_xts,
+	  .encrypt = encrypt_aes_xts
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/keys.c b/xlators/encryption/crypt/src/keys.c
new file mode 100644
index 000000000..4a1d3bb5a
--- /dev/null
+++ b/xlators/encryption/crypt/src/keys.c
@@ -0,0 +1,302 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/* Key hierarchy
+                  
+                   +----------------+
+                   | MASTER_VOL_KEY |
+                   +-------+--------+
+                           |
+                           |
+          +----------------+----------------+
+          |                |                |
+          |                |                |
+  +-------+------+ +-------+-------+ +------+--------+
+  | NMTD_VOL_KEY | | EMTD_FILE_KEY | | DATA_FILE_KEY |
+  +-------+------+ +---------------+ +---------------+
+          |
+          |
+  +-------+-------+
+  | NMTD_LINK_KEY |
+  +---------------+
+
+ */
+
+#if DEBUG_CRYPT
+static void check_prf_iters(uint32_t num_iters)
+{
+	if (num_iters == 0)
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad number of prf iterations : %d", num_iters);
+}
+#else
+#define check_prf_iters(num_iters) noop
+#endif /* DEBUG_CRYPT */
+
+unsigned char crypt_fake_oid[16] =
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+/*
+ * derive key in the counter mode using
+ * sha256-based HMAC as PRF, see
+ * NIST Special Publication 800-108, 5.1)
+ */
+
+#define PRF_OUTPUT_SIZE  SHA256_DIGEST_LENGTH
+
+static int32_t kderive_init(struct kderive_context *ctx,
+			    const unsigned char *pkey, /* parent key */
+			    uint32_t pkey_size,  /* parent key size */
+			    const unsigned char *idctx, /* id-context */
+			    uint32_t idctx_size,
+			    crypt_key_type type  /* type of child key */)
+{
+	unsigned char *pos;
+	uint32_t llen = strlen(crypt_keys[type].label);	
+	/*
+	 * Compoud the fixed input data for KDF:
+	 * [i]_2 || Label || 0x00 || Id-Context || [L]_2),
+	 * NIST SP 800-108, 5.1
+	 */
+	ctx->fid_len =
+		sizeof(uint32_t) +
+		llen +
+		1 +
+		idctx_size +
+		sizeof(uint32_t);
+
+	ctx->fid = GF_CALLOC(ctx->fid_len, 1, gf_crypt_mt_key);
+	if (!ctx->fid)
+		return ENOMEM;
+	ctx->out_len = round_up(crypt_keys[type].len >> 3,
+				PRF_OUTPUT_SIZE);
+	ctx->out = GF_CALLOC(ctx->out_len, 1, gf_crypt_mt_key);
+	if (!ctx->out) {
+		GF_FREE(ctx->fid);
+		return ENOMEM;
+	}
+	ctx->pkey = pkey;
+	ctx->pkey_len = pkey_size;
+	ctx->ckey_len = crypt_keys[type].len;
+
+	pos = ctx->fid;
+
+	/* counter will be set up in kderive_rfn() */
+	pos += sizeof(uint32_t);
+
+	memcpy(pos, crypt_keys[type].label, llen);
+	pos += llen;
+
+	/* set up zero octet */
+	*pos = 0;
+	pos += 1;
+
+	memcpy(pos, idctx, idctx_size);
+	pos += idctx_size;
+
+	*((uint32_t *)pos) = htobe32(ctx->ckey_len);
+
+	return 0;
+}
+
+static void kderive_update(struct kderive_context *ctx)
+{	
+	uint32_t i;
+	HMAC_CTX hctx;
+	unsigned char *pos = ctx->out;
+	uint32_t *p_iter = (uint32_t *)ctx->fid;
+	uint32_t num_iters = ctx->out_len / PRF_OUTPUT_SIZE;
+
+	check_prf_iters(num_iters);
+
+	HMAC_CTX_init(&hctx);
+	for (i = 0; i < num_iters; i++) {
+		/*
+		 * update the iteration number in the fid
+		 */
+		*p_iter = htobe32(i);
+		HMAC_Init_ex(&hctx,
+			     ctx->pkey, ctx->pkey_len >> 3,
+			     EVP_sha256(),
+			     NULL);
+		HMAC_Update(&hctx, ctx->fid, ctx->fid_len);
+		HMAC_Final(&hctx, pos, NULL);
+
+		pos += PRF_OUTPUT_SIZE;
+	}
+	HMAC_CTX_cleanup(&hctx);
+}
+
+static void kderive_final(struct kderive_context *ctx, unsigned char *child)
+{
+	memcpy(child, ctx->out, ctx->ckey_len >> 3);
+	GF_FREE(ctx->fid);
+	GF_FREE(ctx->out);
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+/*
+ * derive per-volume key for object ids aithentication
+ */
+int32_t get_nmtd_vol_key(struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   crypt_fake_oid, sizeof(uuid_t), NMTD_VOL_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, master->m_nmtd_key);
+	return 0;
+}
+
+/*
+ * derive per-link key for aithentication of non-encrypted
+ * meta-data (nmtd)
+ */
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_nmtd_key,
+			   nmtd_vol_key_size(),
+			   (const unsigned char *)loc->path,
+			   strlen(loc->path), NMTD_LINK_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+/*
+ * derive per-file key for encryption and authentication
+ * of encrypted part of metadata (emtd)
+ */
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), EMTD_FILE_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+static int32_t data_key_type_by_size(uint32_t keysize, crypt_key_type *type)
+{
+	int32_t ret = 0;
+	switch (keysize) {
+	case 256:
+		*type = DATA_FILE_KEY_256;
+		break;
+	case 512:
+		*type = DATA_FILE_KEY_512;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Unsupported data key size %d",
+		       keysize);
+		ret = ENOTSUP;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * derive per-file key for data encryption
+ */
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+	crypt_key_type type;
+
+	ret = data_key_type_by_size(keysize, &type);
+	if (ret)
+		return ret;
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), type);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, key);
+	return 0;
+}
+
+/*
+ * NOTE: Don't change existing keys: it will break compatibility;
+ */
+struct crypt_key crypt_keys[LAST_KEY_TYPE] = {
+	[MASTER_VOL_KEY] =
+	{ .len = MASTER_VOL_KEY_SIZE << 3,
+	  .label = "volume-master",
+	},
+	[NMTD_VOL_KEY] =
+	{ .len = NMTD_VOL_KEY_SIZE << 3,
+	  .label = "volume-nmtd-key-generation"
+	},
+	[NMTD_LINK_KEY] =
+	{ .len = 128,
+	  .label = "link-nmtd-authentication"
+	},
+	[EMTD_FILE_KEY] =
+	{ .len = 128,
+	  .label = "file-emtd-encryption-and-auth"
+	},
+	[DATA_FILE_KEY_256] =
+	{ .len = 256,
+	  .label = "file-data-encryption-256"
+	},
+	[DATA_FILE_KEY_512] =
+	{ .len = 512,
+	  .label = "file-data-encryption-512"
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.c b/xlators/encryption/crypt/src/metadata.c
new file mode 100644
index 000000000..36b14c055
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.c
@@ -0,0 +1,605 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+#include "metadata.h"
+
+int32_t alloc_format(crypt_local_t *local, size_t size)
+{
+	if (size > 0) {
+		local->format = GF_CALLOC(1, size, gf_crypt_mt_mtd);
+		if (!local->format)
+			return ENOMEM;
+	}
+	local->format_size = size;
+	return 0;
+}
+
+int32_t alloc_format_create(crypt_local_t *local)
+{
+	return alloc_format(local, new_format_size());
+}
+
+void free_format(crypt_local_t *local)
+{
+	GF_FREE(local->format);
+}
+
+/*
+ * Check compatibility with extracted metadata
+ */
+static int32_t check_file_metadata(struct crypt_inode_info *info)
+{
+	struct object_cipher_info *object = &info->cinfo;
+
+	if (info->nr_minor != CRYPT_XLATOR_ID) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported minor subversion %d", info->nr_minor);
+		return EINVAL;
+	}
+	if (object->o_alg > LAST_CIPHER_ALG) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher algorithm %d",
+		       object->o_alg);
+		return EINVAL;
+	}
+	if (object->o_mode > LAST_CIPHER_MODE) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher mode %d",
+		       object->o_mode);
+		return EINVAL;
+	}
+	if (object->o_block_bits < CRYPT_MIN_BLOCK_BITS ||
+	    object->o_block_bits > CRYPT_MAX_BLOCK_BITS) {
+		gf_log("crypt", GF_LOG_WARNING, "unsupported block bits %d",
+		       object->o_block_bits);
+		return EINVAL;
+	}
+	/* TBD: check data key size */
+	return 0;
+}
+
+static size_t format_size_v1(mtd_op_t op, size_t old_size)
+{
+	
+	switch (op) {
+	case MTD_CREATE:
+		return sizeof(struct mtd_format_v1);
+	case MTD_OVERWRITE:
+		return old_size;
+	case MTD_APPEND:
+		return old_size + NMTD_8_MAC_SIZE;
+	case MTD_CUT:
+		if (old_size > sizeof(struct mtd_format_v1))
+			return old_size - NMTD_8_MAC_SIZE;
+		else
+			return 0;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad mtd operation");
+		return 0;
+	}
+}
+
+/*
+ * Calculate size of the updated format string.
+ * Returned zero means that we don't need to update the format string.
+ */
+size_t format_size(mtd_op_t op, size_t old_size)
+{
+	size_t versioned;
+
+	versioned = mtd_loaders[current_mtd_loader()].format_size(op,
+				 old_size - sizeof(struct crypt_format));
+	if (versioned != 0)
+		return versioned + sizeof(struct crypt_format);
+	return 0;
+}
+
+/*
+ * size of the format string of newly created file (nr_links = 1)
+ */
+size_t new_format_size(void)
+{
+	return format_size(MTD_CREATE, 0);
+}
+
+/*
+ * Calculate per-link MAC by pathname
+ */
+static int32_t calc_link_mac_v1(struct mtd_format_v1 *fmt,
+				loc_t *loc,
+				unsigned char *result,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;	
+	unsigned char nmtd_link_key[16];
+	CMAC_CTX *cctx;
+	size_t len;
+
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not get nmtd link key");
+		return -1;
+	}
+	cctx = CMAC_CTX_new();
+	if (!cctx) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_CTX_new failed");
+		return -1; 
+	}
+	ret = CMAC_Init(cctx, nmtd_link_key, sizeof(nmtd_link_key),
+			EVP_aes_128_cbc(), 0);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Init failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Update(cctx, get_NMTD_V1(info), SIZE_OF_NMTD_V1);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Update failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Final(cctx, result, &len);
+	CMAC_CTX_free(cctx);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Final failed");
+		return -1;
+	}
+	return 0;
+}
+
+/*
+ * Create per-link MAC of index @idx by pathname
+ */
+static int32_t create_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	memcpy(mac, cmac, SIZE_OF_NMTD_V1_MAC);
+	return 0;
+}
+
+static int32_t create_format_v1(unsigned char *wire,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	unsigned char nmtd_link_key[16];
+	uint32_t ad;
+	GCM128_CONTEXT *gctx;
+
+	fmt = (struct mtd_format_v1 *)wire;
+
+	fmt->minor_id = info->nr_minor;
+	fmt->alg_id = AES_CIPHER_ALG;
+	fmt->dkey_factor = master->m_dkey_size >> KEY_FACTOR_BITS;
+	fmt->block_bits = master->m_block_bits;
+	fmt->mode_id = master->m_mode;
+	/*
+	 * retrieve keys for the parts of metadata
+	 */
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret)
+		return ret;
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret)
+		return ret;
+	
+	AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+
+	/* TBD: Check return values */
+
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	ret = CRYPTO_gcm128_encrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_encrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * set MAC of encrypted part of metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC);
+	CRYPTO_gcm128_release(gctx);
+	/*
+	 * set the first MAC of non-encrypted part of metadata 
+	 */
+	return create_link_mac_v1(fmt, 0, loc, info, master);
+}
+
+/*
+ * Called by fops:
+ * ->create();
+ * ->link();
+ *
+ * Pack common and version-specific parts of file's metadata
+ * Pre-conditions: @info contains valid object-id.
+ */
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master)
+{
+	struct crypt_format *fmt = (struct crypt_format *)wire;
+
+	fmt->loader_id = current_mtd_loader();
+
+	wire += sizeof(struct crypt_format);
+	return mtd_loaders[current_mtd_loader()].create_format(wire, loc,
+							       info, master);
+}
+
+/*
+ * Append or overwrite per-link mac of @mac_idx index
+ * in accordance with the new pathname
+ */
+int32_t appov_link_mac_v1(unsigned char *new,
+			  unsigned char *old,
+			  uint32_t old_size,
+			  int32_t mac_idx,
+			  loc_t *loc,
+			  struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  crypt_local_t *local)
+{
+	memcpy(new, old, old_size);
+	return create_link_mac_v1((struct mtd_format_v1 *)new, mac_idx,
+				  loc, info, master);
+}
+
+/*
+ * Cut per-link mac of @mac_idx index
+ */
+static int32_t cut_link_mac_v1(unsigned char *new,
+			       unsigned char *old,			       
+			       uint32_t old_size,
+			       int32_t mac_idx,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local)
+{
+	memcpy(new,
+	       old,
+	       sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1));
+
+	memcpy(new + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1),
+	       old + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx,
+	       old_size - (sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx));
+	return 0;
+}
+
+int32_t update_format_v1(unsigned char *new,
+			 unsigned char *old,
+				size_t old_len,
+				int32_t mac_idx, /* of old name */
+				mtd_op_t op,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master,
+				crypt_local_t *local)
+{
+	switch (op) {
+	case MTD_APPEND:
+		mac_idx = 1 + (old_len - sizeof(struct mtd_format_v1))/8;
+	case MTD_OVERWRITE:
+		return appov_link_mac_v1(new, old, old_len, mac_idx,
+					 loc, info, master, local);
+	case MTD_CUT:
+		return cut_link_mac_v1(new, old, old_len, mac_idx,
+				       loc, info, master, local);
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad  mtd operation %d", op);
+		return -1;
+	}
+}
+
+/*
+ * Called by fops:
+ *
+ * ->link()
+ * ->unlink()
+ * ->rename()
+ *
+ */
+int32_t update_format(unsigned char *new,
+		      unsigned char *old,
+		      size_t old_len,
+		      int32_t mac_idx,
+		      mtd_op_t op,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local)
+{
+	if (!new)
+		return 0;
+	memcpy(new, old, sizeof(struct crypt_format));
+
+	old += sizeof(struct crypt_format);
+	new += sizeof(struct crypt_format);
+	old_len -= sizeof(struct crypt_format);
+
+	return mtd_loaders[current_mtd_loader()].update_format(new, old,
+							       old_len,
+							       mac_idx, op,
+							       loc, info,
+							       master, local);
+}
+
+/*
+ * Perform preliminary checks of found metadata
+ * Return < 0 on errors;
+ * Return number of object-id MACs (>= 1) on success
+ */
+int32_t check_format_v1(uint32_t len, unsigned char *wire)
+{
+	uint32_t nr_links;
+
+	if (len < sizeof(struct mtd_format_v1)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata size %d", len);
+		goto error;
+	}
+	len -= sizeof(struct mtd_format_v1);
+	if (len % sizeof(nmtd_8_mac_t)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata format");
+		goto error;
+	}
+	nr_links = 1 + len / sizeof(nmtd_8_mac_t);
+	if (nr_links > _POSIX_LINK_MAX)
+		goto error;
+	return nr_links;
+ error:
+	return EIO;
+}
+
+/*
+ * Verify per-link MAC specified by index @idx
+ *
+ * return:
+ * -1 on errors;
+ *  0 on failed verification;
+ *  1 on sucessful verification
+ */
+static int32_t verify_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx /* index of the mac to verify */,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	if (memcmp(cmac, mac, SIZE_OF_NMTD_V1_MAC))
+		return 0;
+	return 1;
+}
+
+/*
+ * Lookup per-link MAC by pathname.
+ *
+ * return index of the MAC, if it was found;
+ * return < 0 on errors, or if the MAC wasn't found
+ */
+static int32_t lookup_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t nr_macs,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	uint32_t idx;
+
+	for (idx = 0; idx < nr_macs; idx++) {
+		ret = verify_link_mac_v1(fmt, idx, loc, info, master);
+		if (ret < 0)
+			return ret;
+		if (ret > 0)
+			return idx;
+	}
+	return -ENOENT;
+}
+
+/*
+ * Extract version-specific part of metadata
+ */
+static int32_t open_format_v1(unsigned char *wire,
+			      int32_t len,
+			      loc_t *loc,
+			      struct crypt_inode_info *info,
+			      struct master_cipher_info *master,
+			      crypt_local_t *local,
+			      gf_boolean_t load_info)
+{
+	int32_t ret;
+	int32_t num_nmtd_macs;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	GCM128_CONTEXT *gctx;
+	uint32_t ad;
+	emtd_8_mac_t gmac;
+	struct object_cipher_info *object;
+
+	num_nmtd_macs = check_format_v1(len, wire);
+	if (num_nmtd_macs <= 0)
+		return EIO;
+	fmt = (struct mtd_format_v1 *)wire;
+
+	ret = lookup_link_mac_v1(fmt, num_nmtd_macs, loc, info, master);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "NMTD verification failed");
+		return EINVAL;
+	}
+	local->mac_idx = ret;
+	if (load_info == _gf_false)
+		/* the case of partial open */
+		return 0;
+
+	object = &info->cinfo;
+
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not retrieve metadata key");
+		return ret;
+	}
+	/*
+	 * decrypt encrypted meta-data
+	 */
+	ret = AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not set encrypt key");
+		return ret;
+	}
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+	if (!gctx) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not alloc gcm context");
+		return ENOMEM;
+	}
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;	
+	}
+	ret = CRYPTO_gcm128_decrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_decrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * verify metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, gmac, sizeof(gmac));
+	CRYPTO_gcm128_release(gctx);
+	if (memcmp(gmac, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC)) {
+		gf_log("crypt", GF_LOG_ERROR, "EMTD verification failed");
+		return EINVAL;
+	}
+	/*
+	 * load verified metadata to the private part of inode
+	 */
+	info->nr_minor = fmt->minor_id;
+
+	object->o_alg = fmt->alg_id;
+	object->o_dkey_size = fmt->dkey_factor << KEY_FACTOR_BITS;
+	object->o_block_bits = fmt->block_bits;
+	object->o_mode = fmt->mode_id;
+
+	return check_file_metadata(info);
+}
+
+/*
+ * perform metadata authentication against @loc->path;
+ * extract crypt-specific attribtes and populate @info
+ * with them (optional)
+ */
+int32_t open_format(unsigned char *str,
+		    int32_t len,
+		    loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master,
+		    crypt_local_t *local,
+		    gf_boolean_t load_info)
+{
+	struct crypt_format *fmt;
+	if (len < sizeof(*fmt)) {
+		gf_log("crypt", GF_LOG_ERROR, "Bad core format");
+		return EIO;
+	}
+	fmt = (struct crypt_format *)str;
+
+	if (fmt->loader_id >= LAST_MTD_LOADER) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Unsupported loader id %d", fmt->loader_id);
+		return EINVAL;
+	}
+	str += sizeof(*fmt);
+	len -= sizeof(*fmt);
+
+	return mtd_loaders[fmt->loader_id].open_format(str,
+						       len,
+						       loc,
+						       info,
+						       master,
+						       local,
+						       load_info);
+}
+
+struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER] = {
+	[MTD_LOADER_V1] = 
+	{.format_size = format_size_v1,
+	 .create_format = create_format_v1,
+	 .open_format = open_format_v1,
+	 .update_format = update_format_v1
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.h b/xlators/encryption/crypt/src/metadata.h
new file mode 100644
index 000000000..a92f149ef
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.h
@@ -0,0 +1,74 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __METADATA_H__
+#define __METADATA_H__
+
+#define NMTD_8_MAC_SIZE   (8)
+#define EMTD_8_MAC_SIZE   (8)
+
+typedef uint8_t nmtd_8_mac_t[NMTD_8_MAC_SIZE];
+typedef uint8_t emtd_8_mac_t[EMTD_8_MAC_SIZE] ;
+
+/*
+ * Version "v1" of file's metadata.
+ * Metadata of this version has 4 components:
+ * 
+ * 1) EMTD (Encrypted part of MeTaData);
+ * 2) NMTD (Non-encrypted part of MeTaData);
+ * 3) EMTD_MAC; (EMTD Message Authentication Code);
+ * 4) Array of per-link NMTD MACs (for every (hard)link it includes
+ *    exactly one MAC)
+ */
+struct mtd_format_v1 {
+	/* EMTD, encrypted part of meta-data */
+	uint8_t            alg_id;  /* cipher algorithm id (only AES for now) */
+        uint8_t            mode_id; /* cipher mode id; (only XTS for now) */
+	uint8_t            block_bits; /* encoded block size */
+	uint8_t            minor_id; /* client translator id */
+	uint8_t            dkey_factor; /* encoded size of the data key */
+	/* MACs */
+	emtd_8_mac_t       gmac; /* MAC of the encrypted meta-data, 8 bytes */
+	nmtd_8_mac_t       omac; /* per-link MACs of the non-encrypted
+				  * meta-data: at least one such MAC is always
+				  * present */
+} __attribute__((packed));
+
+/*
+ * NMTD, the non-encrypted part of metadata of version "v1"
+ * is file's gfid, which is generated on trusted machines.
+ */
+#define SIZE_OF_NMTD_V1 (sizeof(uuid_t))
+#define SIZE_OF_EMTD_V1 (offsetof(struct mtd_format_v1, gmac) -		\
+			 offsetof(struct mtd_format_v1, alg_id))
+#define SIZE_OF_NMTD_V1_MAC (NMTD_8_MAC_SIZE)
+#define SIZE_OF_EMTD_V1_MAC (EMTD_8_MAC_SIZE)
+
+static inline unsigned char *get_EMTD_V1(struct mtd_format_v1 *format)
+{
+        return &format->alg_id;
+}
+
+static inline unsigned char *get_NMTD_V1(struct crypt_inode_info *info)
+{
+        return info->oid;
+}
+
+static inline unsigned char *get_EMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->gmac;
+}
+
+static inline unsigned char *get_NMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->omac;
+}
+
+#endif /* __METADATA_H__ */
diff --git a/xlators/encryption/rot-13/src/Makefile.am b/xlators/encryption/rot-13/src/Makefile.am
index ba5e623d8..94e8d18e7 100644
--- a/xlators/encryption/rot-13/src/Makefile.am
+++ b/xlators/encryption/rot-13/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = rot-13.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
 
-rot_13_la_LDFLAGS = -module -avoidversion
+rot_13_la_LDFLAGS = -module -avoid-version
 
 rot_13_la_SOURCES = rot-13.c
 rot_13_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = rot-13.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/encryption/rot-13/src/rot-13.c b/xlators/encryption/rot-13/src/rot-13.c
index 5e6820206..1bcfe0192 100644
--- a/xlators/encryption/rot-13/src/rot-13.c
+++ b/xlators/encryption/rot-13/src/rot-13.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <ctype.h>
 #include <sys/uio.h>
 
@@ -32,13 +22,13 @@
 #include "rot-13.h"
 
 /*
- * This is a rot13 ``encryption'' xlator. It rot13's data when 
- * writing to disk and rot13's it back when reading it. 
+ * This is a rot13 ``encryption'' xlator. It rot13's data when
+ * writing to disk and rot13's it back when reading it.
  * This xlator is meant as an example, NOT FOR PRODUCTION
  * USE ;) (hence no error-checking)
  */
 
-void 
+void
 rot13 (char *buf, int len)
 {
 	int i;
@@ -68,14 +58,15 @@ rot13_readv_cbk (call_frame_t *frame,
                  struct iovec *vector,
                  int32_t count,
 		 struct iatt *stbuf,
-                 struct iobref *iobref)
+                 struct iobref *iobref, dict_t *xdata)
 {
 	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-  
+
 	if (priv->decrypt_read)
 		rot13_iovec (vector, count);
 
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count, stbuf, iobref);
+	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
+                             stbuf, iobref, xdata);
 	return 0;
 }
 
@@ -84,13 +75,13 @@ rot13_readv (call_frame_t *frame,
              xlator_t *this,
              fd_t *fd,
              size_t size,
-             off_t offset, uint32_t flags)
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
 	STACK_WIND (frame,
 		    rot13_readv_cbk,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset, flags);
+		    fd, size, offset, flags, xdata);
 	return 0;
 }
 
@@ -101,9 +92,10 @@ rot13_writev_cbk (call_frame_t *frame,
                   int32_t op_ret,
                   int32_t op_errno,
                   struct iatt *prebuf,
-		  struct iatt *postbuf)
+		  struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 	return 0;
 }
 
@@ -114,7 +106,7 @@ rot13_writev (call_frame_t *frame,
               struct iovec *vector,
               int32_t count,
               off_t offset, uint32_t flags,
-              struct iobref *iobref)
+              struct iobref *iobref, dict_t *xdata)
 {
 	rot_13_private_t *priv = (rot_13_private_t *)this->private;
 	if (priv->encrypt_write)
@@ -125,7 +117,7 @@ rot13_writev (call_frame_t *frame,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->writev,
 		    fd, vector, count, offset, flags,
-                    iobref);
+                    iobref, xdata);
 	return 0;
 }
 
@@ -136,7 +128,7 @@ init (xlator_t *this)
 	rot_13_private_t *priv = NULL;
 
 	if (!this->children || this->children->next) {
-		gf_log ("rot13", GF_LOG_ERROR, 
+		gf_log ("rot13", GF_LOG_ERROR,
 			"FATAL: rot13 should have exactly one child");
 		return -1;
 	}
@@ -158,6 +150,7 @@ init (xlator_t *this)
 		if (gf_string2boolean (data->data, &priv->encrypt_write) == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"encrypt-write takes only boolean options");
+			GF_FREE (priv);
 			return -1;
 		}
 	}
@@ -167,6 +160,7 @@ init (xlator_t *this)
 		if (gf_string2boolean (data->data, &priv->decrypt_read) == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"decrypt-read takes only boolean options");
+			GF_FREE (priv);
 			return -1;
 		}
 	}
@@ -194,15 +188,14 @@ struct xlator_fops fops = {
 	.writev       = rot13_writev
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-	{ .key  = {"encrypt-write"}, 
+	{ .key  = {"encrypt-write"},
 	  .type = GF_OPTION_TYPE_BOOL
 	},
-	{ .key  = {"decrypt-read"}, 
-	  .type = GF_OPTION_TYPE_BOOL 
+	{ .key  = {"decrypt-read"},
+	  .type = GF_OPTION_TYPE_BOOL
 	},
 	{ .key  = {NULL} },
 };
diff --git a/xlators/encryption/rot-13/src/rot-13.h b/xlators/encryption/rot-13/src/rot-13.h
index 8ef8162ae..3e9fc19c7 100644
--- a/xlators/encryption/rot-13/src/rot-13.h
+++ b/xlators/encryption/rot-13/src/rot-13.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __ROT_13_H__
 #define __ROT_13_H__
 
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index 884f82d68..d2f5ef192 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = locks trash quota read-only mac-compat quiesce marker index#path-converter # filter
+SUBDIRS = locks quota read-only mac-compat quiesce marker index \
+	  protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block # trash path-converter # filter
 
 CLEANFILES =
diff --git a/xlators/features/changelog/Makefile.am b/xlators/features/changelog/Makefile.am
new file mode 100644
index 000000000..153bb6850
--- /dev/null
+++ b/xlators/features/changelog/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src lib
+
+CLEANFILES =
diff --git a/xlators/storage/bdb/Makefile.am b/xlators/features/changelog/lib/Makefile.am
index d471a3f92..a985f42a8 100644
--- a/xlators/storage/bdb/Makefile.am
+++ b/xlators/features/changelog/lib/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/changelog/lib/examples/c/get-changes.c b/xlators/features/changelog/lib/examples/c/get-changes.c
new file mode 100644
index 000000000..6d0d0357d
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/c/get-changes.c
@@ -0,0 +1,87 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+/**
+ * get set of new changes every 10 seconds (just print the file names)
+ *
+ * Compile it using:
+ *  gcc -o getchanges `pkg-config --cflags libgfchangelog` get-changes.c \
+ *  `pkg-config --libs libgfchangelog`
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <limits.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <errno.h>
+
+#include "changelog.h"
+
+#define handle_error(fn)                                \
+        printf ("%s (reason: %s)\n", fn, strerror (errno))
+
+int
+main (int argc, char ** argv)
+{
+        int     i           = 0;
+        int     ret         = 0;
+        ssize_t nr_changes  = 0;
+        ssize_t changes     = 0;
+        char fbuf[PATH_MAX] = {0,};
+
+        /* get changes for brick "/home/vshankar/export/yow/yow-1" */
+        ret = gf_changelog_register ("/home/vshankar/exports/yow/yow-1",
+                                     "/tmp/scratch", "/tmp/change.log", 9, 5);
+        if (ret) {
+                handle_error ("register failed");
+                goto out;
+        }
+
+        while (1) {
+                i = 0;
+                nr_changes = gf_changelog_scan ();
+                if (nr_changes < 0) {
+                        handle_error ("scan(): ");
+                        break;
+                }
+
+                if (nr_changes == 0)
+                        goto next;
+
+                printf ("Got %ld changelog files\n", nr_changes);
+
+                while ( (changes =
+                         gf_changelog_next_change (fbuf, PATH_MAX)) > 0) {
+                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
+
+                        /* process changelog */
+                        /* ... */
+                        /* ... */
+                        /* ... */
+                        /* done processing */
+
+                        ret = gf_changelog_done (fbuf);
+                        if (ret)
+                                handle_error ("gf_changelog_done");
+                }
+
+                if (changes == -1)
+                        handle_error ("gf_changelog_next_change");
+
+        next:
+                sleep (10);
+        }
+
+ out:
+        return ret;
+}
diff --git a/xlators/features/changelog/lib/examples/python/changes.py b/xlators/features/changelog/lib/examples/python/changes.py
new file mode 100644
index 000000000..d21db8eab
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/python/changes.py
@@ -0,0 +1,32 @@
+#!/usr/bin/python
+
+import os
+import sys
+import time
+import libgfchangelog
+
+cl = libgfchangelog.Changes()
+
+def get_changes(brick, scratch_dir, log_file, log_level, interval):
+    change_list = []
+    try:
+        cl.cl_register(brick, scratch_dir, log_file, log_level)
+        while True:
+            cl.cl_scan()
+            change_list = cl.cl_getchanges()
+            if change_list:
+                print change_list
+            for change in change_list:
+                print('done with %s' % (change))
+                cl.cl_done(change)
+            time.sleep(interval)
+    except OSError:
+        ex = sys.exc_info()[1]
+        print ex
+
+if __name__ == '__main__':
+    if len(sys.argv) != 5:
+        print("usage: %s <brick> <scratch-dir> <log-file> <fetch-interval>"
+              % (sys.argv[0]))
+        sys.exit(1)
+    get_changes(sys.argv[1], sys.argv[2], sys.argv[3], 9, int(sys.argv[4]))
diff --git a/xlators/features/changelog/lib/examples/python/libgfchangelog.py b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
new file mode 100644
index 000000000..68ec3baf1
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
@@ -0,0 +1,64 @@
+import os
+from ctypes import *
+from ctypes.util import find_library
+
+class Changes(object):
+    libgfc = CDLL(find_library("gfchangelog"), use_errno=True)
+
+    @classmethod
+    def geterrno(cls):
+        return get_errno()
+
+    @classmethod
+    def raise_oserr(cls):
+        errn = cls.geterrno()
+        raise OSError(errn, os.strerror(errn))
+
+    @classmethod
+    def _get_api(cls, call):
+        return getattr(cls.libgfc, call)
+
+    @classmethod
+    def cl_register(cls, brick, path, log_file, log_level, retries = 0):
+        ret = cls._get_api('gf_changelog_register')(brick, path,
+                                                    log_file, log_level, retries)
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_scan(cls):
+        ret = cls._get_api('gf_changelog_scan')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_startfresh(cls):
+        ret = cls._get_api('gf_changelog_start_fresh')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_getchanges(cls):
+        """ remove hardcoding for path name length """
+        def clsort(f):
+            return f.split('.')[-1]
+        changes = []
+        buf = create_string_buffer('\0', 4096)
+        call = cls._get_api('gf_changelog_next_change')
+
+        while True:
+            ret = call(buf, 4096)
+            if ret in (0, -1):
+                break;
+            changes.append(buf.raw[:ret-1])
+        if ret == -1:
+            cls.raise_oserr()
+        # cleanup tracker
+        cls.cl_startfresh()
+        return sorted(changes, key=clsort)
+
+    @classmethod
+    def cl_done(cls, clfile):
+        ret = cls._get_api('gf_changelog_done')(clfile)
+        if ret == -1:
+            cls.raise_oserr()
diff --git a/xlators/features/changelog/lib/src/Makefile.am b/xlators/features/changelog/lib/src/Makefile.am
new file mode 100644
index 000000000..775f026cf
--- /dev/null
+++ b/xlators/features/changelog/lib/src/Makefile.am
@@ -0,0 +1,37 @@
+libgfchangelog_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
+			-DDATADIR=\"$(localstatedir)\"
+
+libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -fpic \
+				-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
+				-I$(top_srcdir)/xlators/features/changelog/src \
+				-DDATADIR=\"$(localstatedir)\"
+
+libgfchangelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+			$(GF_GLUSTERFS_LIBS)
+
+libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS) -version-info $(LIBGFCHANGELOG_LT_VERSION)
+
+libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
+lib_LTLIBRARIES = libgfchangelog.la
+
+CONTRIB_BUILDDIR = $(top_builddir)/contrib
+
+libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-process.c \
+		gf-changelog-helpers.c $(CONTRIBDIR)/uuid/clear.c \
+		$(CONTRIBDIR)/uuid/copy.c $(CONTRIBDIR)/uuid/gen_uuid.c \
+		$(CONTRIBDIR)/uuid/pack.c $(CONTRIBDIR)/uuid/parse.c \
+		$(CONTRIBDIR)/uuid/unparse.c $(CONTRIBDIR)/uuid/uuid_time.c \
+		$(CONTRIBDIR)/uuid/compare.c $(CONTRIBDIR)/uuid/isnull.c \
+		$(CONTRIBDIR)/uuid/unpack.c
+
+noinst_HEADERS = gf-changelog-helpers.h $(CONTRIBDIR)/uuid/uuidd.h \
+		$(CONTRIBDIR)/uuid/uuid.h $(CONTRIBDIR)/uuid/uuidP.h \
+		$(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+
+libgfchangelog_HEADERS = changelog.h
+
+CLEANFILES =
+CONFIG_CLEAN_FILES = $(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+
+$(top_builddir)/libglusterfs/src/libglusterfs.la:
+	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/xlators/features/changelog/lib/src/changelog.h b/xlators/features/changelog/lib/src/changelog.h
new file mode 100644
index 000000000..5cddfb583
--- /dev/null
+++ b/xlators/features/changelog/lib/src/changelog.h
@@ -0,0 +1,31 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_H
+#define _GF_CHANGELOG_H
+
+/* API set */
+
+int
+gf_changelog_register (char *brick_path, char *scratch_dir,
+                       char *log_file, int log_levl, int max_reconnects);
+ssize_t
+gf_changelog_scan ();
+
+int
+gf_changelog_start_fresh ();
+
+ssize_t
+gf_changelog_next_change (char *bufptr, size_t maxlen);
+
+int
+gf_changelog_done (char *file);
+
+#endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.c b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
new file mode 100644
index 000000000..1eef8bf04
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
@@ -0,0 +1,180 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-mem-types.h"
+#include "gf-changelog-helpers.h"
+
+ssize_t gf_changelog_read_path (int fd, char *buffer, size_t bufsize)
+{
+        return read (fd, buffer, bufsize);
+}
+
+size_t
+gf_changelog_write (int fd, char *buffer, size_t len)
+{
+        ssize_t size = 0;
+        size_t writen = 0;
+
+        while (writen < len) {
+                size = write (fd,
+                              buffer + writen, len - writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        return writen;
+}
+
+void
+gf_rfc3986_encode (unsigned char *s, char *enc, char *estr)
+{
+        for (; *s; s++) {
+                if (estr[*s])
+                        sprintf(enc, "%c", estr[*s]);
+                else
+                        sprintf(enc, "%%%02X", *s);
+                while (*++enc);
+        }
+}
+
+/**
+ * thread safe version of readline with buffering
+ * (taken from Unix Network Programming Volume I, W.R. Stevens)
+ *
+ * This is favoured over fgets() as we'd need to ftruncate()
+ * (see gf_changelog_scan() API) to record new changelog files.
+ * stream open functions does have a truncate like api (although
+ * that can be done via @fflush(fp), @ftruncate(fd) and @fseek(fp),
+ * but this involves mixing POSIX file descriptors and stream FILE *).
+ *
+ * NOTE: This implmentation still does work with more than one fd's
+ *       used to perform gf_readline(). For this very reason it's not
+ *       made a part of libglusterfs.
+ */
+
+static pthread_key_t rl_key;
+static pthread_once_t rl_once = PTHREAD_ONCE_INIT;
+
+static void
+readline_destructor (void *ptr)
+{
+        GF_FREE (ptr);
+}
+
+static void
+readline_once (void)
+{
+        pthread_key_create (&rl_key, readline_destructor);
+}
+
+static ssize_t
+my_read (read_line_t *tsd, int fd, char *ptr)
+{
+        if (tsd->rl_cnt <= 0) {
+                if ( (tsd->rl_cnt = read (fd, tsd->rl_buf, MAXLINE)) < 0 )
+                        return -1;
+                else if (tsd->rl_cnt == 0)
+                        return 0;
+                tsd->rl_bufptr = tsd->rl_buf;
+        }
+
+        tsd->rl_cnt--;
+        *ptr = *tsd->rl_bufptr++;
+        return 1;
+}
+
+static int
+gf_readline_init_once (read_line_t **tsd)
+{
+        if (pthread_once (&rl_once, readline_once) != 0)
+                return -1;
+
+        *tsd = pthread_getspecific (rl_key);
+        if (*tsd)
+                goto out;
+
+        *tsd = GF_CALLOC (1, sizeof (**tsd),
+                          gf_changelog_mt_libgfchangelog_rl_t);
+        if (!*tsd)
+                return -1;
+
+        if (pthread_setspecific (rl_key, *tsd) != 0)
+                return -1;
+
+ out:
+        return 0;
+}
+
+ssize_t
+gf_readline (int fd, void *vptr, size_t maxlen)
+{
+        size_t       n   = 0;
+        size_t       rc  = 0;
+        char         c   = ' ';
+        char        *ptr = NULL;
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        ptr = vptr;
+        for (n = 1; n < maxlen; n++) {
+                if ( (rc = my_read (tsd, fd, &c)) == 1 ) {
+                        *ptr++ = c;
+                        if (c == '\n')
+                                break;
+                } else if (rc == 0) {
+                        *ptr = '\0';
+                        return (n - 1);
+                } else
+                        return -1;
+        }
+
+        *ptr = '\0';
+        return n;
+
+}
+
+off_t
+gf_lseek (int fd, off_t offset, int whence)
+{
+        off_t        off = 0;
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        if ( (off = lseek (fd, offset, whence)) == -1)
+                return -1;
+
+        tsd->rl_cnt = 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+
+        return off;
+}
+
+int
+gf_ftruncate (int fd, off_t length)
+{
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        if (ftruncate (fd, 0))
+                return -1;
+
+        tsd->rl_cnt = 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+
+        return 0;
+}
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.h b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
new file mode 100644
index 000000000..7e13d9376
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
@@ -0,0 +1,97 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_HELPERS_H
+#define _GF_CHANGELOG_HELPERS_H
+
+#include <unistd.h>
+#include <dirent.h>
+#include <limits.h>
+#include <pthread.h>
+
+#include <xlator.h>
+
+#define GF_CHANGELOG_TRACKER  "tracker"
+
+#define GF_CHANGELOG_CURRENT_DIR    ".current"
+#define GF_CHANGELOG_PROCESSED_DIR  ".processed"
+#define GF_CHANGELOG_PROCESSING_DIR ".processing"
+
+#ifndef MAXLINE
+#define MAXLINE 4096
+#endif
+
+#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len) do {     \
+                memcpy (ascii + off, ptr, len);                 \
+                off += len;                                     \
+        } while (0)
+
+typedef struct read_line {
+        int rl_cnt;
+        char *rl_bufptr;
+        char rl_buf[MAXLINE];
+} read_line_t;
+
+typedef struct gf_changelog {
+        xlator_t *this;
+
+        /* 'processing' directory stream */
+        DIR *gfc_dir;
+
+        /* fd to the tracker file */
+        int gfc_fd;
+
+        /* connection retries */
+        int gfc_connretries;
+
+        char gfc_sockpath[UNIX_PATH_MAX];
+
+        char gfc_brickpath[PATH_MAX];
+
+        /* socket for recieving notifications */
+        int gfc_sockfd;
+
+        char *gfc_working_dir;
+
+        /* RFC 3986 string encoding */
+        char rfc3986[256];
+
+        char gfc_current_dir[PATH_MAX];
+        char gfc_processed_dir[PATH_MAX];
+        char gfc_processing_dir[PATH_MAX];
+
+        pthread_t gfc_changelog_processor;
+} gf_changelog_t;
+
+int
+gf_changelog_notification_init (xlator_t *this, gf_changelog_t *gfc);
+
+void *
+gf_changelog_process (void *data);
+
+ssize_t
+gf_changelog_read_path (int fd, char *buffer, size_t bufsize);
+
+void
+gf_rfc3986_encode (unsigned char *s, char *enc, char *estr);
+
+size_t
+gf_changelog_write (int fd, char *buffer, size_t len);
+
+ssize_t
+gf_readline (int fd, void *vptr, size_t maxlen);
+
+int
+gf_ftruncate (int fd, off_t length);
+
+off_t
+gf_lseek (int fd, off_t offset, int whence);
+
+#endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-process.c b/xlators/features/changelog/lib/src/gf-changelog-process.c
new file mode 100644
index 000000000..3ea2700c6
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-process.c
@@ -0,0 +1,618 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <pthread.h>
+
+#include "uuid.h"
+#include "globals.h"
+#include "glusterfs.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+
+extern int byebye;
+
+/**
+ * number of gfid records after fop number
+ */
+int nr_gfids[] = {
+        [GF_FOP_MKNOD]   = 1,
+        [GF_FOP_MKDIR]   = 1,
+        [GF_FOP_UNLINK]  = 1,
+        [GF_FOP_RMDIR]   = 1,
+        [GF_FOP_SYMLINK] = 1,
+        [GF_FOP_RENAME]  = 2,
+        [GF_FOP_LINK]    = 1,
+        [GF_FOP_CREATE]  = 1,
+};
+
+int nr_extra_recs[] = {
+        [GF_FOP_MKNOD]   = 3,
+        [GF_FOP_MKDIR]   = 3,
+        [GF_FOP_UNLINK]  = 0,
+        [GF_FOP_RMDIR]   = 0,
+        [GF_FOP_SYMLINK] = 0,
+        [GF_FOP_RENAME]  = 0,
+        [GF_FOP_LINK]    = 0,
+        [GF_FOP_CREATE]  = 3,
+};
+
+static char *
+binary_to_ascii (uuid_t uuid)
+{
+        return uuid_utoa (uuid);
+}
+
+static char *
+conv_noop (char *ptr) { return ptr; }
+
+#define VERIFY_SEPARATOR(ptr, plen, perr)       \
+        {                                       \
+                if (*(ptr + plen) != '\0') {    \
+                        perr = 1;               \
+                        break;                  \
+                }                               \
+        }
+
+#define MOVER_MOVE(mover, nleft, bytes)         \
+        {                                       \
+                mover += bytes;                 \
+                nleft -= bytes;                 \
+        }                                       \
+
+#define PARSE_GFID(mov, ptr, le, fn, perr)                      \
+        {                                                       \
+                VERIFY_SEPARATOR (mov, le, perr);               \
+                ptr = fn (mov);                                 \
+                if (!ptr) {                                     \
+                        perr = 1;                               \
+                        break;                                  \
+                }                                               \
+        }
+
+#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                          \
+        {                                                               \
+                GF_CHANGELOG_FILL_BUFFER (pt, buf, of, strlen (pt));    \
+                MOVER_MOVE (mo, nl, le);                                \
+        }
+
+
+#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)          \
+        {                                                       \
+                memcpy (uuid, mover, sizeof (uuid_t));          \
+                ptr = binary_to_ascii (uuid);                   \
+                if (!ptr) {                                     \
+                        perr = 1;                               \
+                        break;                                  \
+                }                                               \
+                MOVER_MOVE (mover, nleft, sizeof (uuid_t));     \
+        }                                                       \
+
+#define LINE_BUFSIZE  3*PATH_MAX /* enough buffer for extra chars too */
+
+/**
+ * using mmap() makes parsing easy. fgets() cannot be used here as
+ * the binary gfid could contain a line-feed (0x0A), in that case fgets()
+ * would read an incomplete line and parsing would fail. using POSIX fds
+ * would result is additional code to maintain state in case of partial
+ * reads of data (where multiple entries do not fit extirely in the buffer).
+ *
+ * mmap() gives the flexibility of pointing to an offset in the file
+ * without us worrying about reading it in memory (VM does that for us for
+ * free).
+ */
+
+static int
+gf_changelog_parse_binary (xlator_t *this,
+                           gf_changelog_t *gfc, int from_fd, int to_fd,
+                           size_t start_offset, struct stat *stbuf)
+
+{
+        int     ret              = -1;
+        off_t   off              = 0;
+        off_t   nleft            = 0;
+        uuid_t  uuid             = {0,};
+        char   *ptr              = NULL;
+        char   *bname_start      = NULL;
+        char   *bname_end        = NULL;
+        char   *mover            = NULL;
+        char   *start            = NULL;
+        char    current_mover    = ' ';
+        size_t  blen             = 0;
+        int     parse_err        = 0;
+        char ascii[LINE_BUFSIZE] = {0,};
+
+        nleft = stbuf->st_size;
+
+        start = (char *) mmap (NULL, nleft,
+                               PROT_READ, MAP_PRIVATE, from_fd, 0);
+        if (!start) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mmap() error (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        mover = start;
+
+        MOVER_MOVE (mover, nleft, start_offset);
+
+        while (nleft > 0) {
+
+                off = blen = 0;
+                ptr = bname_start = bname_end = NULL;
+
+                current_mover = *mover;
+
+                switch (current_mover) {
+                case 'D':
+                case 'M':
+                        MOVER_MOVE (mover, nleft, 1);
+                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+
+                        break;
+
+                case 'E':
+                        MOVER_MOVE (mover, nleft, 1);
+                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+
+                        bname_start = mover;
+                        if ( (bname_end = strchr (mover, '\n')) == NULL ) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        blen = bname_end - bname_start;
+                        MOVER_MOVE (mover, nleft, blen);
+
+                        break;
+
+                default:
+                        parse_err = 1;
+                }
+
+                if (parse_err)
+                        break;
+
+                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (ptr, ascii, off, strlen (ptr));
+                if (blen)
+                        GF_CHANGELOG_FILL_BUFFER (bname_start,
+                                                  ascii, off, blen);
+                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
+
+                if (gf_changelog_write (to_fd, ascii, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "processing binary changelog failed due to "
+                                " error in writing ascii change (reason: %s)",
+                                strerror (errno));
+                        break;
+                }
+
+                MOVER_MOVE (mover, nleft, 1);
+        }
+
+        if ( (nleft == 0) && (!parse_err))
+                ret = 0;
+
+        if (munmap (start, stbuf->st_size))
+                gf_log (this->name, GF_LOG_ERROR,
+                        "munmap() error (reason: %s)", strerror (errno));
+ out:
+        return ret;
+}
+
+/**
+ * ascii decoder:
+ *  - separate out one entry from another
+ *  - use fop name rather than fop number
+ */
+static int
+gf_changelog_parse_ascii (xlator_t *this,
+                          gf_changelog_t *gfc, int from_fd, int to_fd,
+                          size_t start_offset, struct stat *stbuf)
+{
+        int           ng            = 0;
+        int           ret           = -1;
+        int           fop           = 0;
+        int           len           = 0;
+        off_t         off           = 0;
+        off_t         nleft         = 0;
+        char         *ptr           = NULL;
+        char         *eptr          = NULL;
+        char         *start         = NULL;
+        char         *mover         = NULL;
+        int           parse_err     = 0;
+        char          current_mover = ' ';
+        char ascii[LINE_BUFSIZE]    = {0,};
+        const char   *fopname       = NULL;
+
+        nleft = stbuf->st_size;
+
+        start = (char *) mmap (NULL, nleft,
+                               PROT_READ, MAP_PRIVATE, from_fd, 0);
+        if (!start) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mmap() error (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        mover = start;
+
+        MOVER_MOVE (mover, nleft, start_offset);
+
+        while (nleft > 0) {
+                off = 0;
+                current_mover = *mover;
+
+                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+
+                switch (current_mover) {
+                case 'D':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE(ptr, ascii, off,
+                                      mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        break;
+                case 'M':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE (ptr, ascii, off,
+                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        FILL_AND_MOVE (" ", ascii, off, mover, nleft, 1);
+
+                        /* fop */
+                        len = strlen (mover);
+                        VERIFY_SEPARATOR (mover, len, parse_err);
+
+                        fop = atoi (mover);
+                        if ( (fopname = gf_fop_list[fop]) == NULL) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        MOVER_MOVE (mover, nleft, len);
+
+                        len = strlen (fopname);
+                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
+
+                        break;
+
+                case 'E':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE (ptr, ascii, off,
+                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        FILL_AND_MOVE (" ", ascii, off,
+                                       mover, nleft, 1);
+
+                        /* fop */
+                        len = strlen (mover);
+                        VERIFY_SEPARATOR (mover, len, parse_err);
+
+                        fop = atoi (mover);
+                        if ( (fopname = gf_fop_list[fop]) == NULL) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        MOVER_MOVE (mover, nleft, len);
+
+                        len = strlen (fopname);
+                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
+
+                        ng = nr_extra_recs[fop];
+                        for (;ng > 0; ng--) {
+                                MOVER_MOVE (mover, nleft, 1);
+                                len = strlen (mover);
+                                VERIFY_SEPARATOR (mover, len, parse_err);
+
+                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+                                FILL_AND_MOVE (mover, ascii,
+                                               off, mover, nleft, len);
+                        }
+
+                        /* pargfid + bname */
+                        ng = nr_gfids[fop];
+                        while (ng-- > 0) {
+                                MOVER_MOVE (mover, nleft, 1);
+                                len = strlen (mover);
+                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+
+                                PARSE_GFID (mover, ptr, len,
+                                            conv_noop, parse_err);
+                                eptr = calloc (3, strlen (ptr));
+                                if (!eptr) {
+                                        parse_err = 1;
+                                        break;
+                                }
+
+                                gf_rfc3986_encode ((unsigned char *) ptr,
+                                                   eptr, gfc->rfc3986);
+                                FILL_AND_MOVE (eptr, ascii, off,
+                                               mover, nleft, len);
+                                free (eptr);
+                        }
+
+                        break;
+                default:
+                        parse_err = 1;
+                }
+
+                if (parse_err)
+                        break;
+
+                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
+
+                if (gf_changelog_write (to_fd, ascii, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "processing ascii changelog failed due to "
+                                " error in writing change (reason: %s)",
+                                strerror (errno));
+                        break;
+                }
+
+                MOVER_MOVE (mover, nleft, 1);
+
+        }
+
+        if ( (nleft == 0) && (!parse_err))
+                ret = 0;
+
+        if (munmap (start, stbuf->st_size))
+                gf_log (this->name, GF_LOG_ERROR,
+                        "munmap() error (reason: %s)", strerror (errno));
+
+ out:
+        return ret;
+}
+
+#define COPY_BUFSIZE  8192
+static int
+gf_changelog_copy (xlator_t *this, int from_fd, int to_fd)
+{
+        ssize_t size                  = 0;
+        char   buffer[COPY_BUFSIZE+1] = {0,};
+
+        while (1) {
+                size = read (from_fd, buffer, COPY_BUFSIZE);
+                if (size <= 0)
+                        break;
+
+                if (gf_changelog_write (to_fd,
+                                        buffer, size) != size) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error processing ascii changlog");
+                        size = -1;
+                        break;
+                }
+        }
+
+        return (size < 0 ? -1 : 0);
+}
+
+static int
+gf_changelog_decode (xlator_t *this, gf_changelog_t *gfc, int from_fd,
+                     int to_fd, struct stat *stbuf, int *zerob)
+{
+        int    ret        = -1;
+        int    encoding   = -1;
+        size_t elen       = 0;
+        char buffer[1024] = {0,};
+
+        CHANGELOG_GET_ENCODING (from_fd, buffer, 1024, encoding, elen);
+        if (encoding == -1) /* unknown encoding */
+                goto out;
+
+        if (!CHANGELOG_VALID_ENCODING (encoding))
+                goto out;
+
+        if (elen == stbuf->st_size) {
+                *zerob = 1;
+                goto out;
+        }
+
+        /**
+         * start processing after the header
+         */
+        lseek (from_fd, elen, SEEK_SET);
+
+        switch (encoding) {
+        case CHANGELOG_ENCODE_BINARY:
+                /**
+                 * this ideally should have been a part of changelog-encoders.c
+                 * (ie. part of the changelog translator).
+                 */
+                ret = gf_changelog_parse_binary (this, gfc, from_fd,
+                                                 to_fd, elen, stbuf);
+                break;
+
+        case CHANGELOG_ENCODE_ASCII:
+                ret = gf_changelog_parse_ascii (this, gfc, from_fd,
+                                                to_fd, elen, stbuf);
+                break;
+        default:
+                ret = gf_changelog_copy (this, from_fd, to_fd);
+        }
+
+ out:
+        return ret;
+}
+
+static int
+gf_changelog_consume (xlator_t *this, gf_changelog_t *gfc, char *from_path)
+{
+        int         ret        = -1;
+        int         fd1        = 0;
+        int         fd2        = 0;
+        int         zerob      = 0;
+        struct stat stbuf      = {0,};
+        char dest[PATH_MAX]    = {0,};
+        char to_path[PATH_MAX] = {0,};
+
+        ret = stat (from_path, &stbuf);
+        if (ret || !S_ISREG(stbuf.st_mode)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat failed on changelog file: %s", from_path);
+                goto out;
+        }
+
+        fd1 = open (from_path, O_RDONLY);
+        if (fd1 < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot open changelog file: %s (reason: %s)",
+                        from_path, strerror (errno));
+                goto out;
+        }
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         gfc->gfc_current_dir, basename (from_path));
+        (void) snprintf (dest, PATH_MAX, "%s%s",
+                         gfc->gfc_processing_dir, basename (from_path));
+
+        fd2 = open (to_path, O_CREAT | O_TRUNC | O_RDWR,
+                    S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (fd2 < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot create ascii changelog file %s (reason %s)",
+                        to_path, strerror (errno));
+                goto close_fd;
+        } else {
+                ret = gf_changelog_decode (this, gfc, fd1,
+                                           fd2, &stbuf, &zerob);
+
+                close (fd2);
+
+                if (!ret) {
+                        /* move it to processing on a successfull
+                           decode */
+                        ret = rename (to_path, dest);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "error moving %s to processing dir"
+                                        " (reason: %s)", to_path,
+                                        strerror (errno));
+                }
+
+                /* remove it from .current if it's an empty file */
+                if (zerob) {
+                        ret = unlink (to_path);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not unlink %s (reason: %s",
+                                        to_path, strerror (errno));
+                }
+        }
+
+ close_fd:
+        close (fd1);
+
+ out:
+        return ret;
+}
+
+static char *
+gf_changelog_ext_change (xlator_t *this,
+                         gf_changelog_t *gfc, char *path, size_t readlen)
+{
+        int     alo = 0;
+        int     ret = 0;
+        size_t  len = 0;
+        char   *buf = NULL;
+
+        buf = path;
+        while (len < readlen) {
+                if (*buf == '\0') {
+                        alo = 1;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "processing changelog: %s", path);
+                        ret = gf_changelog_consume (this, gfc, path);
+                }
+
+                if (ret)
+                        break;
+
+                len++; buf++;
+                if (alo) {
+                        alo = 0;
+                        path = buf;
+                }
+        }
+
+        return (ret) ? NULL : path;
+}
+
+void *
+gf_changelog_process (void *data)
+{
+        ssize_t         len      = 0;
+        ssize_t         offlen   = 0;
+        xlator_t       *this     = NULL;
+        char           *sbuf     = NULL;
+        gf_changelog_t *gfc      = NULL;
+        char from_path[PATH_MAX] = {0,};
+
+        gfc = (gf_changelog_t *) data;
+        this = gfc->this;
+
+        pthread_detach (pthread_self());
+
+        for (;;) {
+                len = gf_changelog_read_path (gfc->gfc_sockfd,
+                                              from_path + offlen,
+                                              PATH_MAX - offlen);
+                if (len < 0)
+                        continue; /* ignore it for now */
+
+                if (len == 0) { /* close() from the changelog translator */
+                        gf_log (this->name, GF_LOG_INFO, "close from changelog"
+                                " notification translator.");
+
+                        if (gfc->gfc_connretries != 1) {
+                                if (!gf_changelog_notification_init(this, gfc))
+                                        continue;
+                        }
+
+                        byebye = 1;
+                        break;
+                }
+
+                len += offlen;
+                sbuf = gf_changelog_ext_change (this, gfc, from_path, len);
+                if (!sbuf) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not extract changelog filename");
+                        continue;
+                }
+
+                offlen = 0;
+                if (sbuf != (from_path + len)) {
+                        offlen = from_path + len - sbuf;
+                        memmove (from_path, sbuf, offlen);
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "byebye (%d) from processing thread...", byebye);
+        return NULL;
+}
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
new file mode 100644
index 000000000..4b2b25ad5
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -0,0 +1,516 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+#include <dirent.h>
+#include <stddef.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+#include "changelog-mem-types.h"
+
+int byebye = 0;
+
+static void
+gf_changelog_cleanup (gf_changelog_t *gfc)
+{
+        /* socket */
+        if (gfc->gfc_sockfd != -1)
+                close (gfc->gfc_sockfd);
+        /* tracker fd */
+        if (gfc->gfc_fd != -1)
+                close (gfc->gfc_fd);
+        /* processing dir */
+        if (gfc->gfc_dir)
+                closedir (gfc->gfc_dir);
+
+        if (gfc->gfc_working_dir)
+                free (gfc->gfc_working_dir); /* allocated by realpath */
+}
+
+void
+__attribute__ ((constructor)) gf_changelog_ctor (void)
+{
+        glusterfs_ctx_t *ctx = NULL;
+
+        ctx = glusterfs_ctx_new ();
+        if (!ctx)
+                return;
+
+        if (glusterfs_globals_init (ctx)) {
+                free (ctx);
+                ctx = NULL;
+                return;
+        }
+
+        THIS->ctx = ctx;
+}
+
+void
+__attribute__ ((destructor)) gf_changelog_dtor (void)
+{
+        xlator_t        *this = NULL;
+        glusterfs_ctx_t *ctx  = NULL;
+        gf_changelog_t  *gfc  = NULL;
+
+        this = THIS;
+        if (!this)
+                return;
+
+        ctx = this->ctx;
+        gfc = this->private;
+
+        if (gfc) {
+                gf_changelog_cleanup (gfc);
+                GF_FREE (gfc);
+        }
+
+        if (ctx) {
+                pthread_mutex_destroy (&ctx->lock);
+                free (ctx);
+                ctx = NULL;
+        }
+}
+
+
+static int
+gf_changelog_open_dirs (gf_changelog_t *gfc)
+{
+        int  ret                    = -1;
+        DIR *dir                    = NULL;
+        int  tracker_fd             = 0;
+        char tracker_path[PATH_MAX] = {0,};
+
+        (void) snprintf (gfc->gfc_current_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_CURRENT_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_current_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        (void) snprintf (gfc->gfc_processed_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_PROCESSED_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_processed_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        (void) snprintf (gfc->gfc_processing_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_PROCESSING_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_processing_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        dir = opendir (gfc->gfc_processing_dir);
+        if (!dir) {
+                gf_log ("", GF_LOG_ERROR,
+                        "opendir() error [reason: %s]", strerror (errno));
+                goto out;
+        }
+
+        gfc->gfc_dir = dir;
+
+        (void) snprintf (tracker_path, PATH_MAX,
+                         "%s/"GF_CHANGELOG_TRACKER, gfc->gfc_working_dir);
+
+        tracker_fd = open (tracker_path, O_CREAT | O_APPEND | O_RDWR,
+                           S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (tracker_fd < 0) {
+                closedir (gfc->gfc_dir);
+                ret = -1;
+                goto out;
+        }
+
+        gfc->gfc_fd = tracker_fd;
+        ret = 0;
+ out:
+        return ret;
+}
+
+int
+gf_changelog_notification_init (xlator_t *this, gf_changelog_t *gfc)
+{
+        int                ret    = 0;
+        int                len    = 0;
+        int                tries  = 0;
+        int                sockfd = 0;
+        struct sockaddr_un remote;
+
+        this = gfc->this;
+
+        if (gfc->gfc_sockfd != -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Reconnecting...");
+                close (gfc->gfc_sockfd);
+        }
+
+        sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
+        if (sockfd < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        CHANGELOG_MAKE_SOCKET_PATH (gfc->gfc_brickpath,
+                                    gfc->gfc_sockpath, UNIX_PATH_MAX);
+        gf_log (this->name, GF_LOG_INFO,
+                "connecting to changelog socket: %s (brick: %s)",
+                gfc->gfc_sockpath, gfc->gfc_brickpath);
+
+        remote.sun_family = AF_UNIX;
+        strcpy (remote.sun_path, gfc->gfc_sockpath);
+
+        len = strlen (remote.sun_path) + sizeof (remote.sun_family);
+
+        while (tries < gfc->gfc_connretries) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "connection attempt %d/%d...",
+                        tries + 1, gfc->gfc_connretries);
+
+                /* initiate a connect */
+                if (connect (sockfd, (struct sockaddr *) &remote, len) == 0) {
+                        gfc->gfc_sockfd = sockfd;
+                        break;
+                }
+
+                tries++;
+                sleep (2);
+        }
+
+        if (tries == gfc->gfc_connretries) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not connect to changelog socket!"
+                        " bailing out...");
+                close (sockfd);
+                ret = -1;
+        } else
+                gf_log (this->name, GF_LOG_INFO,
+                        "connection successful");
+
+ out:
+        return ret;
+}
+
+int
+gf_changelog_done (char *file)
+{
+        int             ret    = -1;
+        char           *buffer = NULL;
+        xlator_t       *this   = NULL;
+        gf_changelog_t *gfc    = NULL;
+        char to_path[PATH_MAX] = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        if (!file || !strlen (file))
+                goto out;
+
+        /* make sure 'file' is inside ->gfc_working_dir */
+        buffer = realpath (file, NULL);
+        if (!buffer)
+                goto out;
+
+        if (strncmp (gfc->gfc_working_dir,
+                     buffer, strlen (gfc->gfc_working_dir)))
+                goto out;
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         gfc->gfc_processed_dir, basename (buffer));
+        gf_log (this->name, GF_LOG_DEBUG,
+                "moving %s to processed directory", file);
+        ret = rename (buffer, to_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot move %s to %s (reason: %s)",
+                        file, to_path, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        if (buffer)
+                free (buffer); /* allocated by realpath() */
+        return ret;
+}
+
+/**
+ * @API
+ *  for a set of changelogs, start from the begining
+ */
+int
+gf_changelog_start_fresh ()
+{
+        xlator_t *this = NULL;
+        gf_changelog_t *gfc = NULL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        errno = EINVAL;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        if (gf_ftruncate (gfc->gfc_fd, 0))
+                goto out;
+
+        return 0;
+
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ * return the next changelog file entry. zero means all chanelogs
+ * consumed.
+ */
+ssize_t
+gf_changelog_next_change (char *bufptr, size_t maxlen)
+{
+        ssize_t         size       = 0;
+        int             tracker_fd = 0;
+        xlator_t       *this       = NULL;
+        gf_changelog_t *gfc        = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        tracker_fd = gfc->gfc_fd;
+
+        size = gf_readline (tracker_fd, buffer, maxlen);
+        if (size < 0)
+                goto out;
+        if (size == 0)
+                return 0;
+
+        memcpy (bufptr, buffer, size - 1);
+        *(buffer + size) = '\0';
+
+        return size;
+
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ *  gf_changelog_scan() - scan and generate a list of change entries
+ *
+ * calling this api multiple times (without calling gf_changlog_done())
+ * would result new changelogs(s) being refreshed in the tracker file.
+ * This call also acts as a cancellation point for the consumer.
+ */
+ssize_t
+gf_changelog_scan ()
+{
+        int             ret        = 0;
+        int             tracker_fd = 0;
+        size_t          len        = 0;
+        size_t          off        = 0;
+        xlator_t       *this       = NULL;
+        size_t          nr_entries = 0;
+        gf_changelog_t *gfc        = NULL;
+        struct dirent  *entryp     = NULL;
+        struct dirent  *result     = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        /**
+         * do we need to protect 'byebye' with locks? worst, the
+         * consumer would get notified during next scan().
+         */
+        if (byebye) {
+                errno = ECONNREFUSED;
+                goto out;
+        }
+
+        errno = EINVAL;
+
+        tracker_fd = gfc->gfc_fd;
+
+        if (gf_ftruncate (tracker_fd, 0))
+                goto out;
+
+        len = offsetof(struct dirent, d_name)
+                + pathconf(gfc->gfc_processing_dir, _PC_NAME_MAX) + 1;
+        entryp = GF_CALLOC (1, len,
+                            gf_changelog_mt_libgfchangelog_dirent_t);
+        if (!entryp)
+                goto out;
+
+        rewinddir (gfc->gfc_dir);
+        while (1) {
+                ret = readdir_r (gfc->gfc_dir, entryp, &result);
+                if (ret || !result)
+                        break;
+
+                if ( !strcmp (basename (entryp->d_name), ".")
+                     || !strcmp (basename (entryp->d_name), "..") )
+                        continue;
+
+                nr_entries++;
+
+                GF_CHANGELOG_FILL_BUFFER (gfc->gfc_processing_dir,
+                                          buffer, off,
+                                          strlen (gfc->gfc_processing_dir));
+                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
+                                          off, strlen (entryp->d_name));
+                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
+
+                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error writing changelog filename"
+                                " to tracker file");
+                        break;
+                }
+                off = 0;
+        }
+
+        GF_FREE (entryp);
+
+        if (!result) {
+                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
+                        return nr_entries;
+        }
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ *  gf_changelog_register() - register a client for updates.
+ */
+int
+gf_changelog_register (char *brick_path, char *scratch_dir,
+                       char *log_file, int log_level, int max_reconnects)
+{
+        int             i    = 0;
+        int             ret  = -1;
+        int             errn = 0;
+        xlator_t       *this = NULL;
+        gf_changelog_t *gfc  = NULL;
+
+        this = THIS;
+        if (!this->ctx)
+                goto out;
+
+        errno = ENOMEM;
+
+        gfc = GF_CALLOC (1, sizeof (*gfc),
+                         gf_changelog_mt_libgfchangelog_t);
+        if (!gfc)
+                goto out;
+
+        gfc->this = this;
+
+        gfc->gfc_dir = NULL;
+        gfc->gfc_fd = gfc->gfc_sockfd = -1;
+
+        gfc->gfc_working_dir = realpath (scratch_dir, NULL);
+        if (!gfc->gfc_working_dir) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_changelog_open_dirs (gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not create entries in scratch dir");
+                goto cleanup;
+        }
+
+        /* passing ident as NULL means to use default ident for syslog */
+        if (gf_log_init (this->ctx, log_file, NULL))
+                goto cleanup;
+
+        gf_log_set_loglevel ((log_level == -1) ? GF_LOG_INFO :
+                             log_level);
+
+        gfc->gfc_connretries = (max_reconnects <= 0) ? 1 : max_reconnects;
+        (void) strncpy (gfc->gfc_brickpath, brick_path, PATH_MAX);
+
+        ret = gf_changelog_notification_init (this, gfc);
+        if (ret) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_thread_create (&gfc->gfc_changelog_processor,
+				NULL, gf_changelog_process, gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error creating changelog processor thread"
+                        " new changes won't be recorded!!!");
+                goto cleanup;
+        }
+
+        for (; i < 256; i++) {
+                gfc->rfc3986[i] =
+                        (isalnum(i) || i == '~' ||
+                        i == '-' || i == '.' || i == '_') ? i : 0;
+        }
+
+        ret = 0;
+        this->private = gfc;
+
+        goto out;
+
+ cleanup:
+        gf_changelog_cleanup (gfc);
+        GF_FREE (gfc);
+        this->private = NULL;
+        errno = errn;
+
+ out:
+        return ret;
+}
diff --git a/xlators/features/changelog/src/Makefile.am b/xlators/features/changelog/src/Makefile.am
new file mode 100644
index 000000000..54c21ac21
--- /dev/null
+++ b/xlators/features/changelog/src/Makefile.am
@@ -0,0 +1,19 @@
+xlator_LTLIBRARIES = changelog.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = changelog-helpers.h changelog-mem-types.h changelog-rt.h \
+	changelog-misc.h changelog-encoders.h changelog-notifier.h
+
+changelog_la_LDFLAGS = -module -avoid-version
+
+changelog_la_SOURCES = changelog.c changelog-rt.c changelog-helpers.c \
+	changelog-encoders.c changelog-notifier.c
+changelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC -D_FILE_OFFSET_BITS=64 \
+	-D_GNU_SOURCE -D$(GF_HOST_OS) -shared -nostartfiles -DDATADIR=\"$(localstatedir)\"
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/changelog/src/changelog-encoders.c b/xlators/features/changelog/src/changelog-encoders.c
new file mode 100644
index 000000000..08626ee2f
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-encoders.c
@@ -0,0 +1,197 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "changelog-encoders.h"
+
+size_t
+entry_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char    *tmpbuf = NULL;
+        size_t  bufsz  = 0;
+        struct changelog_entry_fields *ce = NULL;
+
+        ce = (struct changelog_entry_fields *) data;
+
+        if (encode) {
+                tmpbuf = uuid_utoa (ce->cef_uuid);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
+        } else {
+                CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                                       ce->cef_uuid, sizeof (uuid_t));
+        }
+
+        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
+        CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                               ce->cef_bname, strlen (ce->cef_bname));
+        return bufsz;
+}
+
+size_t
+fop_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char buf[10]          = {0,};
+        size_t         bufsz = 0;
+        glusterfs_fop_t fop   = 0;
+
+        fop = *(glusterfs_fop_t *) data;
+
+        if (encode) {
+                (void) snprintf (buf, sizeof (buf), "%d", fop);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, &fop, sizeof (fop));
+
+        return bufsz;
+}
+
+size_t
+number_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        size_t       bufsz = 0;
+        unsigned int nr    = 0;
+        char buf[20]       = {0,};
+
+        nr = *(unsigned int *) data;
+
+        if (encode) {
+                (void) snprintf (buf, sizeof (buf), "%u", nr);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, &nr, sizeof (unsigned int));
+
+        return bufsz;
+}
+
+void
+entry_free_fn (void *data)
+{
+        changelog_opt_t *co = data;
+
+        if (!co)
+                return;
+
+        GF_FREE (co->co_entry.cef_bname);
+}
+
+/**
+ * try to write all data in one shot
+ */
+
+static inline void
+changelog_encode_write_xtra (changelog_log_data_t *cld,
+                             char *buffer, size_t *off, gf_boolean_t encode)
+{
+        int              i      = 0;
+        size_t           offset = 0;
+        void            *data   = NULL;
+        changelog_opt_t *co     = NULL;
+
+        offset = *off;
+
+        co = (changelog_opt_t *) cld->cld_ptr;
+
+        for (; i < cld->cld_xtra_records; i++, co++) {
+                CHANGELOG_FILL_BUFFER (buffer, offset, "\0", 1);
+
+                switch (co->co_type) {
+                case CHANGELOG_OPT_REC_FOP:
+                        data = &co->co_fop;
+                        break;
+                case CHANGELOG_OPT_REC_ENTRY:
+                        data = &co->co_entry;
+                        break;
+                case CHANGELOG_OPT_REC_UINT32:
+                        data = &co->co_uint32;
+                        break;
+                }
+
+                if (co->co_convert)
+                        offset += co->co_convert (data,
+                                                  buffer + offset, encode);
+                else /* no coversion: write it out as it is */
+                        CHANGELOG_FILL_BUFFER (buffer, offset,
+                                               data, co->co_len);
+        }
+
+        *off = offset;
+}
+
+int
+changelog_encode_ascii (xlator_t *this, changelog_log_data_t *cld)
+{
+        size_t            off      = 0;
+        size_t            gfid_len = 0;
+        char             *gfid_str = NULL;
+        char             *buffer   = NULL;
+        changelog_priv_t *priv     = NULL;
+
+        priv = this->private;
+
+        gfid_str = uuid_utoa (cld->cld_gfid);
+        gfid_len = strlen (gfid_str);
+
+        /* extra bytes for decorations */
+        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
+        CHANGELOG_STORE_ASCII (priv, buffer,
+                               off, gfid_str, gfid_len, cld);
+
+        if (cld->cld_xtra_records)
+                changelog_encode_write_xtra (cld, buffer, &off, _gf_true);
+
+        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+
+        return changelog_write_change (priv, buffer, off);
+}
+
+int
+changelog_encode_binary (xlator_t *this, changelog_log_data_t *cld)
+{
+        size_t            off    = 0;
+        char             *buffer = NULL;
+        changelog_priv_t *priv   = NULL;
+
+        priv = this->private;
+
+        /* extra bytes for decorations */
+        buffer = alloca (sizeof (uuid_t) + cld->cld_ptr_len + 10);
+        CHANGELOG_STORE_BINARY (priv, buffer, off, cld->cld_gfid, cld);
+
+        if (cld->cld_xtra_records)
+                changelog_encode_write_xtra (cld, buffer, &off, _gf_false);
+
+        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+
+        return changelog_write_change (priv, buffer, off);
+}
+
+static struct changelog_encoder
+cb_encoder[] = {
+        [CHANGELOG_ENCODE_BINARY] =
+        {
+                .encoder = CHANGELOG_ENCODE_BINARY,
+                .encode = changelog_encode_binary,
+        },
+        [CHANGELOG_ENCODE_ASCII] =
+        {
+                .encoder = CHANGELOG_ENCODE_ASCII,
+                .encode = changelog_encode_ascii,
+        },
+};
+
+void
+changelog_encode_change( changelog_priv_t * priv)
+{
+        priv->ce = &cb_encoder[priv->encode_mode];
+}
diff --git a/xlators/features/changelog/src/changelog-encoders.h b/xlators/features/changelog/src/changelog-encoders.h
new file mode 100644
index 000000000..c5dcc8a77
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-encoders.h
@@ -0,0 +1,48 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_ENCODERS_H
+#define _CHANGELOG_ENCODERS_H
+
+#include "xlator.h"
+#include "defaults.h"
+
+#include "changelog-helpers.h"
+
+#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld) do { \
+                CHANGELOG_FILL_BUFFER (buffer, off,                     \
+                                       priv->maps[cld->cld_type], 1);   \
+                CHANGELOG_FILL_BUFFER (buffer,                          \
+                                       off, gfid, gfid_len);            \
+        } while (0)
+
+#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld) do {          \
+                CHANGELOG_FILL_BUFFER (buffer, off,                     \
+                                       priv->maps[cld->cld_type], 1);   \
+                CHANGELOG_FILL_BUFFER (buffer,                          \
+                                       off, gfid, sizeof (uuid_t));     \
+        } while (0)
+
+size_t
+entry_fn (void *data, char *buffer, gf_boolean_t encode);
+size_t
+fop_fn (void *data, char *buffer, gf_boolean_t encode);
+size_t
+number_fn (void *data, char *buffer, gf_boolean_t encode);
+void
+entry_free_fn (void *data);
+int
+changelog_encode_binary (xlator_t *, changelog_log_data_t *);
+int
+changelog_encode_ascii (xlator_t *, changelog_log_data_t *);
+void
+changelog_encode_change(changelog_priv_t *);
+
+#endif /* _CHANGELOG_ENCODERS_H */
diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c
new file mode 100644
index 000000000..91c43a16c
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-helpers.c
@@ -0,0 +1,696 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+#include "iobuf.h"
+
+#include "changelog-helpers.h"
+#include "changelog-mem-types.h"
+
+#include "changelog-encoders.h"
+#include <pthread.h>
+
+void
+changelog_thread_cleanup (xlator_t *this, pthread_t thr_id)
+{
+        int   ret    = 0;
+        void *retval = NULL;
+
+        /* send a cancel request to the thread */
+        ret = pthread_cancel (thr_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not cancel thread (reason: %s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = pthread_join (thr_id, &retval);
+        if (ret || (retval != PTHREAD_CANCELED)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cancel request not adhered as expected"
+                        " (reason: %s)", strerror (errno));
+        }
+
+ out:
+        return;
+}
+
+inline void *
+changelog_get_usable_buffer (changelog_local_t *local)
+{
+        changelog_log_data_t *cld = NULL;
+
+        if (!local)
+                return NULL;
+
+        cld = &local->cld;
+        if (!cld->cld_iobuf)
+                return NULL;
+
+        return cld->cld_iobuf->ptr;
+}
+
+inline void
+changelog_set_usable_record_and_length (changelog_local_t *local,
+                                        size_t len, int xr)
+{
+        changelog_log_data_t *cld = NULL;
+
+        cld = &local->cld;
+
+        cld->cld_ptr_len = len;
+        cld->cld_xtra_records = xr;
+}
+
+void
+changelog_local_cleanup (xlator_t *xl, changelog_local_t *local)
+{
+        int                   i   = 0;
+        changelog_opt_t      *co  = NULL;
+        changelog_log_data_t *cld = NULL;
+
+        if (!local)
+                return;
+
+        cld = &local->cld;
+
+        /* cleanup dynamic allocation for extra records */
+        if (cld->cld_xtra_records) {
+                co = (changelog_opt_t *) cld->cld_ptr;
+                for (; i < cld->cld_xtra_records; i++, co++)
+                        if (co->co_free)
+                                co->co_free (co);
+        }
+
+        CHANGELOG_IOBUF_UNREF (cld->cld_iobuf);
+
+        if (local->inode)
+                inode_unref (local->inode);
+
+        mem_put (local);
+}
+
+inline int
+changelog_write (int fd, char *buffer, size_t len)
+{
+        ssize_t size = 0;
+        size_t writen = 0;
+
+        while (writen < len) {
+                size = write (fd,
+                              buffer + writen, len - writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        return (writen != len);
+}
+
+static int
+changelog_rollover_changelog (xlator_t *this,
+                              changelog_priv_t *priv, unsigned long ts)
+{
+        int   ret            = -1;
+        int   notify         = 0;
+        char *bname          = NULL;
+        char ofile[PATH_MAX] = {0,};
+        char nfile[PATH_MAX] = {0,};
+
+        if (priv->changelog_fd != -1) {
+                close (priv->changelog_fd);
+                priv->changelog_fd = -1;
+        }
+
+        (void) snprintf (ofile, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME, priv->changelog_dir);
+        (void) snprintf (nfile, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME".%lu",
+                         priv->changelog_dir, ts);
+
+        ret = rename (ofile, nfile);
+        if (!ret)
+                notify = 1;
+
+        if (ret && (errno == ENOENT)) {
+                ret = 0;
+        }
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error renaming %s -> %s (reason %s)",
+                        ofile, nfile, strerror (errno));
+        }
+
+        if (notify) {
+                bname = basename (nfile);
+                gf_log (this->name, GF_LOG_DEBUG, "notifying: %s", bname);
+                ret = changelog_write (priv->wfd, bname, strlen (bname) + 1);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to send file name to notify thread"
+                                " (reason: %s)", strerror (errno));
+                }
+        }
+
+        return ret;
+}
+
+int
+changelog_open (xlator_t *this,
+                changelog_priv_t *priv)
+{
+        int fd                        = 0;
+        int ret                       = -1;
+        int flags                     = 0;
+        char buffer[1024]             = {0,};
+        char changelog_path[PATH_MAX] = {0,};
+
+        (void) snprintf (changelog_path, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME,
+                         priv->changelog_dir);
+
+        flags |= (O_CREAT | O_RDWR);
+        if (priv->fsync_interval == 0)
+                flags |= O_SYNC;
+
+        fd = open (changelog_path, flags,
+                   S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to open/create changelog file %s"
+                        " (reason: %s). change-logging will be"
+                        " inactive", changelog_path, strerror (errno));
+                goto out;
+        }
+
+        priv->changelog_fd = fd;
+
+        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
+                         CHANGELOG_VERSION_MAJOR,
+                         CHANGELOG_VERSION_MINOR,
+                         priv->ce->encoder);
+        ret = changelog_write_change (priv, buffer, strlen (buffer));
+        if (ret) {
+                close (priv->changelog_fd);
+                priv->changelog_fd = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int
+changelog_start_next_change (xlator_t *this,
+                             changelog_priv_t *priv,
+                             unsigned long ts, gf_boolean_t finale)
+{
+        int ret = -1;
+
+        ret = changelog_rollover_changelog (this, priv, ts);
+
+        if (!ret && !finale)
+                ret = changelog_open (this, priv);
+
+        return ret;
+}
+
+/**
+ * return the length of entry
+ */
+inline size_t
+changelog_entry_length ()
+{
+        return sizeof (changelog_log_data_t);
+}
+
+int
+changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last)
+{
+        struct timeval tv = {0,};
+
+        cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
+
+        if (gettimeofday (&tv, NULL))
+                return -1;
+
+        cld->cld_roll_time = (unsigned long) tv.tv_sec;
+        cld->cld_finale = is_last;
+        return 0;
+}
+
+int
+changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+{
+        return changelog_write (priv->changelog_fd, buffer, len);
+}
+
+inline int
+changelog_handle_change (xlator_t *this,
+                         changelog_priv_t *priv, changelog_log_data_t *cld)
+{
+        int ret = 0;
+
+        if (CHANGELOG_TYPE_IS_ROLLOVER (cld->cld_type)) {
+                changelog_encode_change(priv);
+                ret = changelog_start_next_change (this, priv,
+                                                   cld->cld_roll_time,
+                                                   cld->cld_finale);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Problem rolling over changelog(s)");
+                goto out;
+        }
+
+        /**
+         * case when there is reconfigure done (disabling changelog) and there
+         * are still fops that have updates in prgress.
+         */
+        if (priv->changelog_fd == -1)
+                return 0;
+
+        if (CHANGELOG_TYPE_IS_FSYNC (cld->cld_type)) {
+                ret = fsync (priv->changelog_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "fsync failed (reason: %s)",
+                                strerror (errno));
+                }
+                goto out;
+        }
+
+        ret = priv->ce->encode (this, cld);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error writing changelog to disk");
+        }
+
+ out:
+        return ret;
+}
+
+changelog_local_t *
+changelog_local_init (xlator_t *this, inode_t *inode,
+                      uuid_t gfid, int xtra_records,
+                      gf_boolean_t update_flag)
+{
+        changelog_local_t *local = NULL;
+        struct iobuf      *iobuf = NULL;
+
+        /**
+         * We relax the presence of inode if @update_flag is true.
+         * The caller (implmentation of the fop) needs to be careful to
+         * not blindly use local->inode.
+         */
+        if (!update_flag && !inode) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                  "inode needed for version checking !!!");
+                goto out;
+        }
+
+        if (xtra_records) {
+                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
+                                    xtra_records * CHANGELOG_OPT_RECORD_LEN);
+                if (!iobuf)
+                        goto out;
+        }
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                CHANGELOG_IOBUF_UNREF (iobuf);
+                goto out;
+        }
+
+        local->update_no_check = update_flag;
+
+        uuid_copy (local->cld.cld_gfid, gfid);
+
+        local->cld.cld_iobuf = iobuf;
+        local->cld.cld_xtra_records = 0; /* set by the caller */
+
+        if (inode)
+                local->inode = inode_ref (inode);
+
+ out:
+        return local;
+}
+
+int
+changelog_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        changelog_inode_ctx_t *ctx = NULL;
+
+        inode_ctx_del (inode, this, &ctx_addr);
+        if (!ctx_addr)
+                return 0;
+
+        ctx = (changelog_inode_ctx_t *) (long) ctx_addr;
+        GF_FREE (ctx);
+
+        return 0;
+}
+
+int
+changelog_inject_single_event (xlator_t *this,
+                               changelog_priv_t *priv,
+                               changelog_log_data_t *cld)
+{
+        return priv->cd.dispatchfn (this, priv, priv->cd.cd_data, cld, NULL);
+}
+
+/**
+ * TODO: these threads have many thing in common (wake up after
+ * a certain time etc..). move them into separate routine.
+ */
+void *
+changelog_rollover (void *data)
+{
+        int                     ret   = 0;
+        xlator_t               *this  = NULL;
+        struct timeval          tv    = {0,};
+        changelog_log_data_t    cld   = {0,};
+        changelog_time_slice_t *slice = NULL;
+        changelog_priv_t       *priv  = data;
+
+        this = priv->cr.this;
+        slice = &priv->slice;
+
+        while (1) {
+                tv.tv_sec  = priv->rollover_time;
+                tv.tv_usec = 0;
+
+                ret = select (0, NULL, NULL, NULL, &tv);
+                if (ret)
+                        continue;
+
+                ret = changelog_fill_rollover_data (&cld, _gf_false);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to fill rollover data");
+                        continue;
+                }
+
+                LOCK (&priv->lock);
+                {
+                        ret = changelog_inject_single_event (this, priv, &cld);
+                        if (!ret)
+                                SLICE_VERSION_UPDATE (slice);
+                }
+                UNLOCK (&priv->lock);
+        }
+
+        return NULL;
+}
+
+void *
+changelog_fsync_thread (void *data)
+{
+        int                   ret  = 0;
+        xlator_t             *this = NULL;
+        struct timeval        tv   = {0,};
+        changelog_log_data_t  cld  = {0,};
+        changelog_priv_t     *priv = data;
+
+        this = priv->cf.this;
+        cld.cld_type = CHANGELOG_TYPE_FSYNC;
+
+        while (1) {
+                tv.tv_sec  = priv->fsync_interval;
+                tv.tv_usec = 0;
+
+                ret = select (0, NULL, NULL, NULL, &tv);
+                if (ret)
+                        continue;
+
+                ret = changelog_inject_single_event (this, priv, &cld);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to inject fsync event");
+        }
+
+        return NULL;
+}
+
+/* macros for inode/changelog version checks */
+
+#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type) do {       \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        LOCK (&priv->lock);                             \
+                        {                                               \
+                                *iver = slice->changelog_version[type]; \
+                        }                                               \
+                        UNLOCK (&priv->lock);                           \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd) do {    \
+                LOCK (&priv->lock);                                     \
+                {                                                       \
+                        upd = (ver == slice->changelog_version[type])   \
+                                ? _gf_false : _gf_true;                 \
+                }                                                       \
+                UNLOCK (&priv->lock);                                   \
+        } while (0)
+
+static int
+__changelog_inode_ctx_set (xlator_t *this,
+                           inode_t *inode, changelog_inode_ctx_t *ctx)
+{
+        uint64_t ctx_addr = (uint64_t) ctx;
+        return __inode_ctx_set (inode, this, &ctx_addr);
+}
+
+/**
+ * one shot routine to get the address and the value of a inode version
+ * for a particular type.
+ */
+static changelog_inode_ctx_t *
+__changelog_inode_ctx_get (xlator_t *this,
+                           inode_t *inode, unsigned long **iver,
+                           unsigned long *version, changelog_log_type type)
+{
+        int                    ret      = 0;
+        uint64_t               ctx_addr = 0;
+        changelog_inode_ctx_t *ctx      = NULL;
+
+        ret = __inode_ctx_get (inode, this, &ctx_addr);
+        if (ret < 0)
+                ctx_addr = 0;
+        if (ctx_addr != 0) {
+                ctx = (changelog_inode_ctx_t *) (long)ctx_addr;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_changelog_mt_inode_ctx_t);
+        if (!ctx)
+                goto out;
+
+        ret = __changelog_inode_ctx_set (this, inode, ctx);
+        if (ret) {
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+
+ out:
+        if (ctx && iver && version) {
+                *iver = CHANGELOG_INODE_VERSION_TYPE (ctx, type);
+                *version = **iver;
+        }
+
+        return ctx;
+}
+
+static changelog_inode_ctx_t *
+changelog_inode_ctx_get (xlator_t *this,
+                         inode_t *inode, unsigned long **iver,
+                         unsigned long *version, changelog_log_type type)
+{
+        changelog_inode_ctx_t *ctx = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __changelog_inode_ctx_get (this,
+                                                 inode, iver, version, type);
+        }
+        UNLOCK (&inode->lock);
+
+        return ctx;
+}
+
+/**
+ * This is the main update routine. Locking has been made granular so as to
+ * maximize parallelism of fops - I'll try to explain it below using execution
+ * timelines.
+ *
+ * Basically, the contention is between multiple execution threads of this
+ * routine and the roll-over thread. So, instead of having a big lock, we hold
+ * granular locks: inode->lock and priv->lock. Now I'll explain what happens
+ * when there is an update and a roll-over at just about the same time.
+ * NOTE:
+ *  - the dispatcher itself synchronizes updates via it's own lock
+ *  - the slice version in incremented by the roll-over thread
+ *
+ * Case 1: When the rollover thread wins before the inode version can be
+ * compared with the slice version.
+ *
+ *          [updater]                 |             [rollover]
+ *                                    |
+ *                                    |           <SLICE: 1, 1, 1>
+ * <changelog_update>                 |
+ *   <changelog_inode_ctx_get>        |
+ *      <CTX: 1, 1, 1>                |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 2, 2, 2>
+ *                                    |         UNLOCK (&priv->lock)
+ *                                    |
+ * LOCK (&priv->lock)                 |
+ *   <INODE_VERSION_EQUALS_SLICE>     |
+ *    I: 1 <-> S: 2                   |
+ *    update: true                    |
+ * UNLOCK (&priv->lock)               |
+ *                                    |
+ * <if update == true>                |
+ *  <dispath-update-event>            |
+ *  <INODE_VERSION_UPDATE>            |
+ *   LOCK (&inode->lock)              |
+ *    LOCK (&priv->lock)              |
+ *     <CTX: 2, 1, 1>                 |
+ *    UNLOCK (&priv->lock)            |
+ *   UNLOCK (&inode->lock)            |
+ *
+ * Therefore, the change gets recorded in the next change (no lost change). If
+ * the slice version was ahead of the inode version (say I:1, S: 2), then
+ * anyway the comparison would result in a update (I: 1, S: 3).
+ *
+ * If the rollover time is too less, then there is another contention when the
+ * updater tries to bring up inode version to the slice version (this is also
+ * the case when the roll-over thread wakes up during INODE_VERSION_UPDATE.
+ *
+ *   <CTX: 1, 1, 1>                   |       <SLICE: 2, 2, 2>
+ *                                    |
+ *                                    |
+ * <dispath-update-event>             |
+ * <INODE_VERSION_UPDATE>             |
+ *  LOCK (&inode->lock)               |
+ *   LOCK (&priv->lock)               |
+ *    <CTX: 2, 1, 1>                  |
+ *   UNLOCK (&priv->lock)             |
+ *  UNLOCK (&inode->lock)             |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 3, 3, 3>
+ *                                    |         UNLOCK (&priv->lock)
+ *
+ *
+ * Case 2: When the fop thread wins
+ *
+ *          [updater]                 |             [rollover]
+ *                                    |
+ *                                    |           <SLICE: 1, 1, 1>
+ * <changelog_update>                 |
+ *   <changelog_inode_ctx_get>        |
+ *      <CTX: 0, 0, 0>                |
+ *                                    |
+ * LOCK (&priv->lock)                 |
+ *   <INODE_VERSION_EQUALS_SLICE>     |
+ *    I: 0 <-> S: 1                   |
+ *    update: true                    |
+ * UNLOCK (&priv->lock)               |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 2, 2, 2>
+ *                                    |         UNLOCK (&priv->lock)
+ * <if update == true>                |
+ *  <dispath-update-event>            |
+ *  <INODE_VERSION_UPDATE>            |
+ *   LOCK (&inode->lock)              |
+ *    LOCK (&priv->lock)              |
+ *     <CTX: 2, 0, 0>                 |
+ *    UNLOCK (&priv->lock)            |
+ *   UNLOCK (&inode->lock)            |
+ *
+ * Here again, if the inode version was equal to the slice version (I: 1, S: 1)
+ * then there is no need to record an update (as the equality of the two version
+ * signifies an update was recorded in the current time slice).
+ */
+inline void
+changelog_update (xlator_t *this, changelog_priv_t *priv,
+                  changelog_local_t *local, changelog_log_type type)
+{
+        int                     ret        = 0;
+        unsigned long          *iver       = NULL;
+        unsigned long           version    = 0;
+        inode_t                *inode      = NULL;
+        changelog_time_slice_t *slice      = NULL;
+        changelog_inode_ctx_t  *ctx        = NULL;
+        changelog_log_data_t   *cld_0      = NULL;
+        changelog_log_data_t   *cld_1      = NULL;
+        changelog_local_t      *next_local = NULL;
+        gf_boolean_t            need_upd   = _gf_true;
+
+        slice = &priv->slice;
+
+        /**
+         * for fops that do not require inode version checking
+         */
+        if (local->update_no_check)
+                goto update;
+
+        inode = local->inode;
+
+        ctx = changelog_inode_ctx_get (this,
+                                       inode, &iver, &version, type);
+        if (!ctx)
+                goto update;
+
+        INODE_VERSION_EQUALS_SLICE (priv, version, slice, type, need_upd);
+
+ update:
+        if (need_upd) {
+                cld_0 = &local->cld;
+                cld_0->cld_type = type;
+
+                if ( (next_local = local->prev_entry) != NULL ) {
+                        cld_1 = &next_local->cld;
+                        cld_1->cld_type = type;
+                }
+
+                ret = priv->cd.dispatchfn (this, priv,
+                                           priv->cd.cd_data, cld_0, cld_1);
+
+                /**
+                 * update after the dispatcher has successfully done
+                 * it's job.
+                 */
+                if (!local->update_no_check && iver && !ret)
+                        INODE_VERSION_UPDATE (priv, inode, iver, slice, type);
+        }
+
+        return;
+}
diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h
new file mode 100644
index 000000000..b441f7068
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-helpers.h
@@ -0,0 +1,405 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_HELPERS_H
+#define _CHANGELOG_HELPERS_H
+
+#include "locking.h"
+#include "timer.h"
+#include "pthread.h"
+#include "iobuf.h"
+
+#include "changelog-misc.h"
+
+/**
+ * the changelog entry
+ */
+typedef struct changelog_log_data {
+        /* rollover related */
+        unsigned long cld_roll_time;
+
+        /* reopen changelog? */
+        gf_boolean_t cld_finale;
+
+        changelog_log_type cld_type;
+
+        /**
+         * sincd gfid is _always_ a necessity, it's not a part
+         * of the iobuf. by doing this we do not add any overhead
+         * for data and metadata related fops.
+         */
+        uuid_t        cld_gfid;
+
+        /**
+         * iobufs are used for optionals records: pargfid, path,
+         * write offsets etc.. It's the fop implementers job
+         * to allocate (iobuf_get() in the fop) and get unref'ed
+         * in the callback (CHANGELOG_STACK_UNWIND).
+         */
+        struct iobuf *cld_iobuf;
+
+#define cld_ptr cld_iobuf->ptr
+
+        /**
+         * after allocation you can point this to the length of
+         * usable data, but make sure it does not exceed the
+         * the size of the requested iobuf.
+         */
+        size_t        cld_iobuf_len;
+
+#define cld_ptr_len cld_iobuf_len
+
+        /**
+         * number of optional records
+         */
+        int cld_xtra_records;
+} changelog_log_data_t;
+
+/**
+ * holder for dispatch function and private data
+ */
+
+typedef struct changelog_priv changelog_priv_t;
+
+typedef struct changelog_dispatcher {
+        void *cd_data;
+        int (*dispatchfn) (xlator_t *, changelog_priv_t *, void *,
+                           changelog_log_data_t *, changelog_log_data_t *);
+} changelog_dispatcher_t;
+
+struct changelog_bootstrap {
+        changelog_mode_t mode;
+        int (*ctor) (xlator_t *, changelog_dispatcher_t *);
+        int (*dtor) (xlator_t *, changelog_dispatcher_t *);
+};
+
+struct changelog_encoder {
+        changelog_encoder_t encoder;
+        int (*encode) (xlator_t *, changelog_log_data_t *);
+};
+
+
+/* xlator private */
+
+typedef struct changelog_time_slice {
+        /**
+         * just in case we need nanosecond granularity some day.
+         * field is unused as of now (maybe we'd need it later).
+         */
+        struct timeval tv_start;
+
+        /**
+         * version of changelog file, incremented each time changes
+         * rollover.
+         */
+        unsigned long changelog_version[CHANGELOG_MAX_TYPE];
+} changelog_time_slice_t;
+
+typedef struct changelog_rollover {
+        /* rollover thread */
+        pthread_t rollover_th;
+
+        xlator_t *this;
+} changelog_rollover_t;
+
+typedef struct changelog_fsync {
+        /* fsync() thread */
+        pthread_t fsync_th;
+
+        xlator_t *this;
+} changelog_fsync_t;
+
+# define CHANGELOG_MAX_CLIENTS  5
+typedef struct changelog_notify {
+        /* reader end of the pipe */
+        int rfd;
+
+        /* notifier thread */
+        pthread_t notify_th;
+
+        /* unique socket path */
+        char sockpath[UNIX_PATH_MAX];
+
+        int socket_fd;
+
+        /**
+         * simple array of accept()'ed fds. Not scalable at all
+         * for large number of clients, but it's okay as we have
+         * a ahrd limit in this version (@CHANGELOG_MAX_CLIENTS).
+         */
+        int client_fd[CHANGELOG_MAX_CLIENTS];
+
+        xlator_t *this;
+} changelog_notify_t;
+
+struct changelog_priv {
+        gf_boolean_t active;
+
+        /* to generate unique socket file per brick */
+        char *changelog_brick;
+
+        /* logging directory */
+        char *changelog_dir;
+
+        /* one file for all changelog types */
+        int changelog_fd;
+
+        gf_lock_t lock;
+
+        /* writen end of the pipe */
+        int wfd;
+
+        /* rollover time */
+        int32_t rollover_time;
+
+        /* fsync() interval */
+        int32_t fsync_interval;
+
+        /* changelog type maps */
+        const char *maps[CHANGELOG_MAX_TYPE];
+
+        /* time slicer */
+        changelog_time_slice_t slice;
+
+        /* context of the updater */
+        changelog_dispatcher_t cd;
+
+        /* context of the rollover thread */
+        changelog_rollover_t cr;
+
+        /* context of fsync thread */
+        changelog_fsync_t cf;
+
+        /* context of the notifier thread */
+        changelog_notify_t cn;
+
+        /* operation mode */
+        changelog_mode_t op_mode;
+
+        /* bootstrap routine for 'current' logger */
+        struct changelog_bootstrap *cb;
+
+        /* encoder mode */
+        changelog_encoder_t encode_mode;
+
+        /* encoder */
+        struct changelog_encoder *ce;
+};
+
+struct changelog_local {
+        inode_t              *inode;
+        gf_boolean_t          update_no_check;
+
+        changelog_log_data_t  cld;
+
+        /**
+         * ->prev_entry is used in cases when there needs to be
+         * additional changelog entry for the parent (eg. rename)
+         * It's analogous to ->next in single linked list world,
+         * but we call it as ->prev_entry... ha ha ha
+         */
+        struct changelog_local *prev_entry;
+};
+
+typedef struct changelog_local changelog_local_t;
+
+/* inode version is stored in inode ctx */
+typedef struct changelog_inode_ctx {
+        unsigned long iversion[CHANGELOG_MAX_TYPE];
+} changelog_inode_ctx_t;
+
+#define CHANGELOG_INODE_VERSION_TYPE(ctx, type)  &(ctx->iversion[type])
+
+/**
+ * Optional Records:
+ *  fops that need to save additional information request a array of
+ *  @changelog_opt_t struct. The array is allocated via @iobufs.
+ */
+typedef enum {
+        CHANGELOG_OPT_REC_FOP,
+        CHANGELOG_OPT_REC_ENTRY,
+        CHANGELOG_OPT_REC_UINT32,
+} changelog_optional_rec_type_t;
+
+struct changelog_entry_fields {
+        uuid_t  cef_uuid;
+        char   *cef_bname;
+};
+
+typedef struct {
+        /**
+         * @co_covert can be used to do post-processing of the record before
+         * it's persisted to the CHANGELOG. If this is NULL, then the record
+         * is persisted as per it's in memory format.
+         */
+        size_t (*co_convert) (void *data, char *buffer, gf_boolean_t encode);
+
+        /* release routines */
+        void (*co_free) (void *data);
+
+        /* type of the field */
+        changelog_optional_rec_type_t co_type;
+
+        /**
+         * sizeof of the 'valid' field in the union. This field is not used if
+         * @co_convert is specified.
+         */
+        size_t co_len;
+
+        union {
+                unsigned int                  co_uint32;
+                glusterfs_fop_t               co_fop;
+                struct changelog_entry_fields co_entry;
+        };
+} changelog_opt_t;
+
+#define CHANGELOG_OPT_RECORD_LEN  sizeof (changelog_opt_t)
+
+/**
+ * helpers routines
+ */
+
+void
+changelog_thread_cleanup (xlator_t *this, pthread_t thr_id);
+inline void *
+changelog_get_usable_buffer (changelog_local_t *local);
+inline void
+changelog_set_usable_record_and_length (changelog_local_t *local,
+                                        size_t len, int xr);
+void
+changelog_local_cleanup (xlator_t *xl, changelog_local_t *local);
+changelog_local_t *
+changelog_local_init (xlator_t *this, inode_t *inode, uuid_t gfid,
+                      int xtra_records, gf_boolean_t update_flag);
+int
+changelog_start_next_change (xlator_t *this,
+                             changelog_priv_t *priv,
+                             unsigned long ts, gf_boolean_t finale);
+int
+changelog_open (xlator_t *this, changelog_priv_t *priv);
+int
+changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last);
+int
+changelog_inject_single_event (xlator_t *this,
+                               changelog_priv_t *priv,
+                               changelog_log_data_t *cld);
+inline size_t
+changelog_entry_length ();
+inline int
+changelog_write (int fd, char *buffer, size_t len);
+int
+changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+inline int
+changelog_handle_change (xlator_t *this,
+                         changelog_priv_t *priv, changelog_log_data_t *cld);
+inline void
+changelog_update (xlator_t *this, changelog_priv_t *priv,
+                  changelog_local_t *local, changelog_log_type type);
+void *
+changelog_rollover (void *data);
+void *
+changelog_fsync_thread (void *data);
+int
+changelog_forget (xlator_t *this, inode_t *inode);
+
+/* macros */
+
+#define CHANGELOG_STACK_UNWIND(fop, frame, params ...) do {             \
+                changelog_local_t *__local = NULL;                      \
+                xlator_t          *__xl    = NULL;                      \
+                if (frame) {                                            \
+                        __local      = frame->local;                    \
+                        __xl         = frame->this;                     \
+                        frame->local = NULL;                            \
+                }                                                       \
+                STACK_UNWIND_STRICT (fop, frame, params);               \
+                changelog_local_cleanup (__xl, __local);                \
+                if (__local && __local->prev_entry)                     \
+                        changelog_local_cleanup (__xl,                  \
+                                                 __local->prev_entry);  \
+        } while (0)
+
+#define CHANGELOG_IOBUF_REF(iobuf) do {         \
+                if (iobuf)                      \
+                        iobuf_ref (iobuf);      \
+        } while (0)
+
+#define CHANGELOG_IOBUF_UNREF(iobuf) do {       \
+                if (iobuf)                      \
+                        iobuf_unref (iobuf);    \
+        } while (0)
+
+#define CHANGELOG_FILL_BUFFER(buffer, off, val, len) do {       \
+                memcpy (buffer + off, val, len);                \
+                off += len;                                     \
+        } while (0)
+
+#define SLICE_VERSION_UPDATE(slice) do {                \
+                int i = 0;                              \
+                for (; i < CHANGELOG_MAX_TYPE; i++) {   \
+                        slice->changelog_version[i]++;  \
+                }                                       \
+        } while (0)
+
+#define CHANGELOG_FILL_UINT32(co, number, converter, xlen) do { \
+                co->co_convert = converter;                     \
+                co->co_free = NULL;                             \
+                co->co_type = CHANGELOG_OPT_REC_UINT32;         \
+                co->co_uint32 = number;                         \
+                xlen += sizeof (unsigned int);                 \
+        } while (0)
+
+#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen) do { \
+                co->co_convert = converter;                     \
+                co->co_free = NULL;                             \
+                co->co_type = CHANGELOG_OPT_REC_FOP;            \
+                co->co_fop = fop;                               \
+                xlen += sizeof (fop);                           \
+        } while (0)
+
+#define CHANGELOG_FILL_ENTRY(co, pargfid, bname,                        \
+                             converter, freefn, xlen, label)            \
+        do {                                                            \
+                co->co_convert = converter;                             \
+                co->co_free = freefn;                                   \
+                co->co_type = CHANGELOG_OPT_REC_ENTRY;                  \
+                uuid_copy (co->co_entry.cef_uuid, pargfid);             \
+                co->co_entry.cef_bname = gf_strdup(bname);              \
+                if (!co->co_entry.cef_bname)                            \
+                        goto label;                                     \
+                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));     \
+        } while (0)
+
+#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                  \
+        local = changelog_local_init (this, inode, gfid, xrec, _gf_false)
+
+#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)          \
+        local = changelog_local_init (this, inode, gfid, xrec, _gf_true)
+
+#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label) do { \
+                if (!priv->active)                              \
+                        goto label;                             \
+                /* ignore rebalance process's activity. */      \
+                if (frame->root->pid == GF_CLIENT_PID_DEFRAG)   \
+                        goto label;                             \
+        } while (0)
+
+/* ignore internal fops */
+#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(dict, label) do {           \
+                if (dict && dict_get (dict, GLUSTERFS_INTERNAL_FOP_KEY)) \
+                        goto label;                                     \
+        } while (0)
+
+#define CHANGELOG_COND_GOTO(priv, cond, label) do {                    \
+                if (!priv->active || cond)                             \
+                        goto label;                                    \
+        } while (0)
+
+#endif /* _CHANGELOG_HELPERS_H */
diff --git a/xlators/features/changelog/src/changelog-mem-types.h b/xlators/features/changelog/src/changelog-mem-types.h
new file mode 100644
index 000000000..d72464eab
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-mem-types.h
@@ -0,0 +1,29 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_MEM_TYPES_H
+#define _CHANGELOG_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_changelog_mem_types {
+        gf_changelog_mt_priv_t                  = gf_common_mt_end + 1,
+        gf_changelog_mt_str_t                   = gf_common_mt_end + 2,
+        gf_changelog_mt_batch_t                 = gf_common_mt_end + 3,
+        gf_changelog_mt_rt_t                    = gf_common_mt_end + 4,
+        gf_changelog_mt_inode_ctx_t             = gf_common_mt_end + 5,
+        gf_changelog_mt_libgfchangelog_t        = gf_common_mt_end + 6,
+        gf_changelog_mt_libgfchangelog_rl_t     = gf_common_mt_end + 7,
+        gf_changelog_mt_libgfchangelog_dirent_t = gf_common_mt_end + 8,
+        gf_changelog_mt_changelog_buffer_t      = gf_common_mt_end + 9,
+        gf_changelog_mt_end
+};
+
+#endif
diff --git a/xlators/features/changelog/src/changelog-misc.h b/xlators/features/changelog/src/changelog-misc.h
new file mode 100644
index 000000000..127b03e2e
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-misc.h
@@ -0,0 +1,101 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_MISC_H
+#define _CHANGELOG_MISC_H
+
+#include "glusterfs.h"
+#include "common-utils.h"
+
+#define CHANGELOG_MAX_TYPE  3
+#define CHANGELOG_FILE_NAME "CHANGELOG"
+
+#define CHANGELOG_VERSION_MAJOR  1
+#define CHANGELOG_VERSION_MINOR  1
+
+#define CHANGELOG_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/changelog-%s.sock"
+
+/**
+ * header starts with the version and the format of the changelog.
+ * 'version' not much of a use now.
+ */
+#define CHANGELOG_HEADER                                                \
+        "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
+
+#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len) do {      \
+                char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};             \
+                md5_wrapper((unsigned char *) brick_path,               \
+                            strlen(brick_path),                         \
+                            md5_sum);                                   \
+                (void) snprintf (sockpath, len,                         \
+                                 CHANGELOG_UNIX_SOCK, md5_sum);         \
+        } while (0)
+
+/**
+ * ... used by libgfchangelog.
+ */
+#define CHANGELOG_GET_ENCODING(fd, buffer, len, enc, enc_len) do {      \
+                FILE *fp;                                               \
+                int fd_dup, maj, min;                                   \
+                                                                        \
+                enc = -1;                                               \
+                fd_dup = dup (fd);                                      \
+                                                                        \
+                if (fd_dup != -1) {                                     \
+                        fp = fdopen (fd_dup, "r");                      \
+                        if (fp) {                                       \
+                                if (fgets (buffer, len, fp)) {          \
+                                        elen = strlen (buffer);         \
+                                        sscanf (buffer,                 \
+                                                CHANGELOG_HEADER,       \
+                                                &maj, &min, &enc);      \
+                                }                                       \
+                                fclose (fp);                            \
+                        } else {                                        \
+                                close (fd_dup);                         \
+                        }                                               \
+                }                                                       \
+        } while (0)
+
+/**
+ * everything after 'CHANGELOG_TYPE_ENTRY' are internal types
+ * (ie. none of the fops trigger this type of event), hence
+ * CHANGELOG_MAX_TYPE = 3
+ */
+typedef enum {
+        CHANGELOG_TYPE_DATA = 0,
+        CHANGELOG_TYPE_METADATA,
+        CHANGELOG_TYPE_ENTRY,
+        CHANGELOG_TYPE_ROLLOVER,
+        CHANGELOG_TYPE_FSYNC,
+} changelog_log_type;
+
+/* operation modes - RT for now */
+typedef enum {
+        CHANGELOG_MODE_RT = 0,
+} changelog_mode_t;
+
+/* encoder types */
+
+typedef enum {
+        CHANGELOG_ENCODE_MIN = 0,
+        CHANGELOG_ENCODE_BINARY,
+        CHANGELOG_ENCODE_ASCII,
+        CHANGELOG_ENCODE_MAX,
+} changelog_encoder_t;
+
+#define CHANGELOG_VALID_ENCODING(enc)                                   \
+        (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
+
+#define CHANGELOG_TYPE_IS_ENTRY(type)  (type == CHANGELOG_TYPE_ENTRY)
+#define CHANGELOG_TYPE_IS_ROLLOVER(type)  (type == CHANGELOG_TYPE_ROLLOVER)
+#define CHANGELOG_TYPE_IS_FSYNC(type)  (type == CHANGELOG_TYPE_FSYNC)
+
+#endif /* _CHANGELOG_MISC_H */
diff --git a/xlators/features/changelog/src/changelog-notifier.c b/xlators/features/changelog/src/changelog-notifier.c
new file mode 100644
index 000000000..5f3d063a8
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-notifier.c
@@ -0,0 +1,314 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-notifier.h"
+
+#include <pthread.h>
+
+inline static void
+changelog_notify_clear_fd (changelog_notify_t *cn, int i)
+{
+        cn->client_fd[i] = -1;
+}
+
+inline static void
+changelog_notify_save_fd (changelog_notify_t *cn, int i, int fd)
+{
+        cn->client_fd[i] = fd;
+}
+
+static int
+changelog_notify_insert_fd (xlator_t *this, changelog_notify_t *cn, int fd)
+{
+        int i   = 0;
+        int ret = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        break;
+        }
+
+        if (i == CHANGELOG_MAX_CLIENTS) {
+                /**
+                 * this case should not be hit as listen() would limit
+                 * the number of completely established connections.
+                 */
+                gf_log (this->name, GF_LOG_WARNING,
+                        "hit max client limit (%d)", CHANGELOG_MAX_CLIENTS);
+                ret = -1;
+        }
+        else
+                changelog_notify_save_fd (cn, i, fd);
+
+        return ret;
+}
+
+static void
+changelog_notify_fill_rset (changelog_notify_t *cn, fd_set *rset, int *maxfd)
+{
+        int i = 0;
+
+        FD_ZERO (rset);
+
+        FD_SET (cn->socket_fd, rset);
+        *maxfd = cn->socket_fd;
+
+        FD_SET (cn->rfd, rset);
+        *maxfd = max (*maxfd, cn->rfd);
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] != -1) {
+                        FD_SET (cn->client_fd[i], rset);
+                        *maxfd = max (*maxfd, cn->client_fd[i]);
+                }
+        }
+
+        *maxfd = *maxfd + 1;
+}
+
+static int
+changelog_notify_client (changelog_notify_t *cn, char *path, ssize_t len)
+{
+        int i = 0;
+        int ret = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        continue;
+
+                if (changelog_write (cn->client_fd[i],
+                                     path, len)) {
+                        ret = -1;
+
+                        close (cn->client_fd[i]);
+                        changelog_notify_clear_fd (cn, i);
+                }
+        }
+
+        return ret;
+}
+
+static void
+changelog_notifier_init (changelog_notify_t *cn)
+{
+        int i = 0;
+
+        cn->socket_fd = -1;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                changelog_notify_clear_fd (cn, i);
+        }
+}
+
+static void
+changelog_close_client_conn (changelog_notify_t *cn)
+{
+        int i = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        continue;
+
+                close (cn->client_fd[i]);
+                changelog_notify_clear_fd (cn, i);
+        }
+}
+
+static void
+changelog_notifier_cleanup (void *arg)
+{
+        changelog_notify_t *cn = NULL;
+
+        cn = (changelog_notify_t *) arg;
+
+        changelog_close_client_conn (cn);
+
+        if (cn->socket_fd != -1)
+                close (cn->socket_fd);
+
+        if (cn->rfd)
+                close (cn->rfd);
+
+        if (unlink (cn->sockpath))
+                gf_log ("", GF_LOG_WARNING,
+                        "could not unlink changelog socket file"
+                        " %s (reason: %s", cn->sockpath, strerror (errno));
+}
+
+void *
+changelog_notifier (void *data)
+{
+        int                 i         = 0;
+        int                 fd        = 0;
+        int                 max_fd    = 0;
+        int                 len       = 0;
+        ssize_t             readlen   = 0;
+        xlator_t           *this      = NULL;
+        changelog_priv_t   *priv      = NULL;
+        changelog_notify_t *cn        = NULL;
+        struct sockaddr_un  local     = {0,};
+        char path[PATH_MAX]           = {0,};
+        char abspath[PATH_MAX]        = {0,};
+
+        char buffer;
+        fd_set rset;
+
+        priv = (changelog_priv_t *) data;
+
+        cn = &priv->cn;
+        this = cn->this;
+
+        pthread_cleanup_push (changelog_notifier_cleanup, cn);
+
+        changelog_notifier_init (cn);
+
+        cn->socket_fd = socket (AF_UNIX, SOCK_STREAM, 0);
+        if (cn->socket_fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "changelog socket error (reason: %s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
+                                    cn->sockpath, UNIX_PATH_MAX);
+        if (unlink (cn->sockpath) < 0) {
+                if (errno != ENOENT) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not unlink changelog socket file (%s)"
+                                " (reason: %s)",
+                                CHANGELOG_UNIX_SOCK, strerror (errno));
+                        goto cleanup;
+                }
+        }
+
+        local.sun_family = AF_UNIX;
+        strcpy (local.sun_path, cn->sockpath);
+
+        len = strlen (local.sun_path) + sizeof (local.sun_family);
+
+        /* bind to the unix domain socket */
+        if (bind (cn->socket_fd, (struct sockaddr *) &local, len) < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not bind to changelog socket (reason: %s)",
+                        strerror (errno));
+                goto cleanup;
+        }
+
+        /* listen for incoming connections */
+        if (listen (cn->socket_fd, CHANGELOG_MAX_CLIENTS) < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "listen() error on changelog socket (reason: %s)",
+                        strerror (errno));
+                goto cleanup;
+        }
+
+        /**
+         * simple select() on all to-be-read file descriptors. This method
+         * though old school works pretty well when you have a handfull of
+         * fd's to be watched (clients).
+         *
+         * Future TODO: move this to epoll based notification facility if
+         *              number of clients increase.
+         */
+        for (;;) {
+                changelog_notify_fill_rset (cn, &rset, &max_fd);
+
+                if (select (max_fd, &rset, NULL, NULL, NULL) < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "select() returned -1 (reason: %s)",
+                                strerror (errno));
+                        sleep (2);
+                        continue;
+                }
+
+                if (FD_ISSET (cn->socket_fd, &rset)) {
+                        fd = accept (cn->socket_fd, NULL, NULL);
+                        if (fd < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "accept error on changelog socket"
+                                        " (reason: %s)", strerror (errno));
+                        } else if (changelog_notify_insert_fd (this, cn, fd)) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "hit max client limit");
+                        }
+                }
+
+                if (FD_ISSET (cn->rfd, &rset)) {
+                        /**
+                         * read changelog filename and notify all connected
+                         * clients.
+                         */
+                        readlen = 0;
+                        while (readlen < PATH_MAX) {
+                                len = read (cn->rfd, &path[readlen++], 1);
+                                if (len == -1) {
+                                        break;
+                                }
+
+                                if (len == 0) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "rollover thread sent EOF"
+                                                " on pipe - possibly a crash.");
+                                        /* be blunt and close all connections */
+                                        pthread_exit(NULL);
+                                }
+
+                                if (path[readlen - 1] == '\0')
+                                        break;
+                        }
+
+                        /* should we close all client connections here too? */
+                        if (len < 0 || readlen == PATH_MAX) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not get pathname from rollover"
+                                        " thread or pathname too long");
+                                goto process_rest;
+                        }
+
+                        (void) snprintf (abspath, PATH_MAX,
+                                         "%s/%s", priv->changelog_dir, path);
+                        if (changelog_notify_client (cn, abspath,
+                                                     strlen (abspath) + 1))
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not notify some clients with new"
+                                        " changelogs");
+                }
+
+        process_rest:
+                for (i = 0; i < CHANGELOG_MAX_CLIENTS; i++) {
+                        if ( (fd = cn->client_fd[i]) == -1 )
+                                continue;
+
+                        if (FD_ISSET (fd, &rset)) {
+                                /**
+                                 * the only data we accept from the client is a
+                                 * disconnect. Anything else is treated as bogus
+                                 * and is silently discarded (also warned!!!).
+                                 */
+                                if ( (readlen = read (fd, &buffer, 1)) <= 0 ) {
+                                        close (fd);
+                                        changelog_notify_clear_fd (cn, i);
+                                } else {
+                                        /* silently discard data and log */
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "misbehaving changelog client");
+                                }
+                        }
+                }
+
+        }
+
+ cleanup:;
+        pthread_cleanup_pop (1);
+
+ out:
+        return NULL;
+}
diff --git a/xlators/features/changelog/src/changelog-notifier.h b/xlators/features/changelog/src/changelog-notifier.h
new file mode 100644
index 000000000..55e728356
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-notifier.h
@@ -0,0 +1,19 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_NOTIFIER_H
+#define _CHANGELOG_NOTIFIER_H
+
+#include "changelog-helpers.h"
+
+void *
+changelog_notifier (void *data);
+
+#endif
diff --git a/xlators/features/changelog/src/changelog-rt.c b/xlators/features/changelog/src/changelog-rt.c
new file mode 100644
index 000000000..c147f68ca
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-rt.c
@@ -0,0 +1,72 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "changelog-rt.h"
+#include "changelog-mem-types.h"
+
+int
+changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd)
+{
+        changelog_rt_t *crt = NULL;
+
+        crt = GF_CALLOC (1, sizeof (*crt),
+                         gf_changelog_mt_rt_t);
+        if (!crt)
+                return -1;
+
+        LOCK_INIT (&crt->lock);
+
+        cd->cd_data = crt;
+        cd->dispatchfn = &changelog_rt_enqueue;
+
+        return 0;
+}
+
+int
+changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd)
+{
+        changelog_rt_t *crt = NULL;
+
+        crt = cd->cd_data;
+
+        LOCK_DESTROY (&crt->lock);
+        GF_FREE (crt);
+
+        return 0;
+}
+
+int
+changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
+{
+        int             ret = 0;
+        changelog_rt_t *crt = NULL;
+
+        crt = (changelog_rt_t *) cbatch;
+
+        LOCK (&crt->lock);
+        {
+                ret = changelog_handle_change (this, priv, cld_0);
+                if (!ret && cld_1)
+                        ret = changelog_handle_change (this, priv, cld_1);
+        }
+        UNLOCK (&crt->lock);
+
+        return ret;
+}
diff --git a/xlators/features/changelog/src/changelog-rt.h b/xlators/features/changelog/src/changelog-rt.h
new file mode 100644
index 000000000..1fc2bbc5b
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-rt.h
@@ -0,0 +1,33 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_RT_H
+#define _CHANGELOG_RT_H
+
+#include "locking.h"
+#include "timer.h"
+#include "pthread.h"
+
+#include "changelog-helpers.h"
+
+/* unused as of now - may be you would need it later */
+typedef struct changelog_rt {
+        gf_lock_t lock;
+} changelog_rt_t;
+
+int
+changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd);
+int
+changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd);
+int
+changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
+
+#endif /* _CHANGELOG_RT_H */
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
new file mode 100644
index 000000000..5fe3b4362
--- /dev/null
+++ b/xlators/features/changelog/src/changelog.c
@@ -0,0 +1,1568 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+#include "iobuf.h"
+
+#include "changelog-rt.h"
+
+#include "changelog-encoders.h"
+#include "changelog-mem-types.h"
+
+#include <pthread.h>
+
+#include "changelog-notifier.h"
+
+static struct changelog_bootstrap
+cb_bootstrap[] = {
+        {
+                .mode = CHANGELOG_MODE_RT,
+                .ctor = changelog_rt_init,
+                .dtor = changelog_rt_fini,
+        },
+};
+
+/* Entry operations - TYPE III */
+
+/**
+ * entry operations do not undergo inode version checking.
+ */
+
+/* {{{ */
+
+/* rmdir */
+
+int32_t
+changelog_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                                preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_rmdir (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, int xflags, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local,
+                                NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_rmdir_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
+                    loc, xflags, xdata);
+        return 0;
+}
+
+/* unlink */
+
+int32_t
+changelog_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                                preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_unlink (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, int xflags, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (xdata, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_unlink_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
+                    loc, xflags, xdata);
+        return 0;
+}
+
+/* rename */
+
+int32_t
+changelog_rename_cbk (call_frame_t *frame,
+                      void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *buf, struct iatt *preoldparent,
+                      struct iatt *postoldparent, struct iatt *prenewparent,
+                      struct iatt *postnewparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (rename, frame, op_ret, op_errno,
+                                buf, preoldparent, postoldparent,
+                                prenewparent, postnewparent, xdata);
+        return 0;
+}
+
+
+int32_t
+changelog_rename (call_frame_t *frame, xlator_t *this,
+                  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        size_t            xtra_len  = 0;
+        changelog_priv_t *priv      = NULL;
+        changelog_opt_t  *co        = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        /* 3 == fop + oldloc + newloc */
+        CHANGELOG_INIT_NOCHECK (this, frame->local,
+                                NULL, oldloc->inode->gfid, 3);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, oldloc->pargfid, oldloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 3);
+
+ wind:
+        STACK_WIND (frame, changelog_rename_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+/* link */
+
+int32_t
+changelog_link_cbk (call_frame_t *frame,
+                    void *cookie, xlator_t *this, int32_t op_ret,
+                    int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (link, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_link (call_frame_t *frame,
+                xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (xdata, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, oldloc->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_link_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+/* mkdir */
+
+int32_t
+changelog_mkdir_cbk (call_frame_t *frame,
+                     void *cookie, xlator_t *this, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_mkdir (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, S_IFDIR | mode, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
+
+ wind:
+        STACK_WIND (frame, changelog_mkdir_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+        return 0;
+}
+
+/* symlink */
+
+int32_t
+changelog_symlink_cbk (call_frame_t *frame,
+                       void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (symlink, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_symlink (call_frame_t *frame, xlator_t *this,
+                   const char *linkname, loc_t *loc,
+                   mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        size_t            xtra_len = 0;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_symlink_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+}
+
+/* mknod */
+
+int32_t
+changelog_mknod_cbk (call_frame_t *frame,
+                     void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (mknod, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_mknod (call_frame_t *frame,
+                 xlator_t *this, loc_t *loc,
+                 mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, mode, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
+
+ wind:
+        STACK_WIND (frame, changelog_mknod_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
+                    loc, mode, dev, umask, xdata);
+        return 0;
+}
+
+/* creat */
+
+int32_t
+changelog_create_cbk (call_frame_t *frame,
+                      void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      fd_t *fd, inode_t *inode, struct iatt *buf,
+                      struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (create, frame,
+                                op_ret, op_errno, fd, inode,
+                                buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_create (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, int32_t flags, mode_t mode,
+                  mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        changelog_opt_t  *co       = NULL;
+        changelog_priv_t *priv     = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        /* init with two extra records */
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
+        if (!frame->local)
+                goto wind;
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, mode, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
+
+ wind:
+        STACK_WIND (frame, changelog_create_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+}
+
+/* }}} */
+
+
+/* Metadata modification fops - TYPE II */
+
+/* {{{ */
+
+/* {f}setattr */
+
+int32_t
+changelog_fsetattr_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, struct iatt *preop_stbuf,
+                        struct iatt *postop_stbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                                preop_stbuf, postop_stbuf, xdata);
+
+        return 0;
+
+
+}
+
+int32_t
+changelog_fsetattr (call_frame_t *frame,
+                    xlator_t *this, fd_t *fd,
+                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 1);
+        if (!frame->local)
+                goto wind;
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+
+ wind:
+        STACK_WIND (frame, changelog_fsetattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+        return 0;
+
+
+}
+
+int32_t
+changelog_setattr_cbk (call_frame_t *frame,
+                       void *cookie, xlator_t *this, int32_t op_ret,
+                       int32_t op_errno, struct iatt *preop_stbuf,
+                       struct iatt *postop_stbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (setattr, frame, op_ret, op_errno,
+                                preop_stbuf, postop_stbuf, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_setattr (call_frame_t *frame,
+                   xlator_t *this, loc_t *loc,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 1);
+        if (!frame->local)
+                goto wind;
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+
+ wind:
+        STACK_WIND (frame, changelog_setattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+        return 0;
+}
+
+/* {f}removexattr */
+
+int32_t
+changelog_fremovexattr_cbk (call_frame_t *frame,
+                            void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fremovexattr (call_frame_t *frame, xlator_t *this,
+                        fd_t *fd, const char *name, dict_t *xdata)
+{
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 1);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+
+ wind:
+        STACK_WIND (frame, changelog_fremovexattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+int32_t
+changelog_removexattr_cbk (call_frame_t *frame,
+                           void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_removexattr (call_frame_t *frame, xlator_t *this,
+                       loc_t *loc, const char *name, dict_t *xdata)
+{
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 1);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+
+ wind:
+        STACK_WIND (frame, changelog_removexattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+/* {f}setxattr */
+
+int32_t
+changelog_setxattr_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_setxattr (call_frame_t *frame,
+                    xlator_t *this, loc_t *loc,
+                    dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 1);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+
+ wind:
+        STACK_WIND (frame, changelog_setxattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+changelog_fsetxattr_cbk (call_frame_t *frame,
+                         void *cookie, xlator_t *this, int32_t op_ret,
+                         int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fsetxattr (call_frame_t *frame,
+                     xlator_t *this, fd_t *fd, dict_t *dict,
+                     int32_t flags, dict_t *xdata)
+{
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 1);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+
+ wind:
+        STACK_WIND (frame, changelog_fsetxattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+}
+
+/* }}} */
+
+
+/* Data modification fops - TYPE I */
+
+/* {{{ */
+
+/* {f}truncate() */
+
+int32_t
+changelog_truncate_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (truncate, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_truncate (call_frame_t *frame,
+                    xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_truncate_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
+                    loc, offset, xdata);
+        return 0;
+}
+
+int32_t
+changelog_ftruncate_cbk (call_frame_t *frame,
+                         void *cookie, xlator_t *this, int32_t op_ret,
+                         int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (ftruncate, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_ftruncate (call_frame_t *frame,
+                     xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_ftruncate_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
+                    fd, offset, xdata);
+        return 0;
+}
+
+/* writev() */
+
+int32_t
+changelog_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf,
+                    dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret <= 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (writev, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_writev (call_frame_t *frame,
+                  xlator_t *this, fd_t *fd, struct iovec *vector,
+                  int32_t count, off_t offset, uint32_t flags,
+                  struct iobref *iobref, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_writev_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev, fd, vector,
+                    count, offset, flags, iobref, xdata);
+        return 0;
+}
+
+/* }}} */
+
+/**
+ * The
+ *   - @init ()
+ *   - @fini ()
+ *   - @reconfigure ()
+ *   ... and helper routines
+ */
+
+/**
+ * needed if there are more operation modes in the future.
+ */
+static void
+changelog_assign_opmode (changelog_priv_t *priv, char *mode)
+{
+        if ( strncmp (mode, "realtime", 8) == 0 ) {
+                priv->op_mode = CHANGELOG_MODE_RT;
+        }
+}
+
+static void
+changelog_assign_encoding (changelog_priv_t *priv, char *enc)
+{
+        if ( strncmp (enc, "binary", 6) == 0 ) {
+                priv->encode_mode = CHANGELOG_ENCODE_BINARY;
+        } else if ( strncmp (enc, "ascii", 5) == 0 ) {
+                priv->encode_mode = CHANGELOG_ENCODE_ASCII;
+        }
+}
+
+/* cleanup any helper threads that are running */
+static void
+changelog_cleanup_helper_threads (xlator_t *this, changelog_priv_t *priv)
+{
+        if (priv->cr.rollover_th) {
+                changelog_thread_cleanup (this, priv->cr.rollover_th);
+                priv->cr.rollover_th = 0;
+        }
+
+        if (priv->cf.fsync_th) {
+                changelog_thread_cleanup (this, priv->cf.fsync_th);
+                priv->cf.fsync_th = 0;
+        }
+}
+
+/* spawn helper thread; cleaning up in case of errors */
+static int
+changelog_spawn_helper_threads (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret = 0;
+
+        priv->cr.this = this;
+        ret = gf_thread_create (&priv->cr.rollover_th,
+				NULL, changelog_rollover, priv);
+        if (ret)
+                goto out;
+
+        if (priv->fsync_interval) {
+                priv->cf.this = this;
+                ret = gf_thread_create (&priv->cf.fsync_th,
+					NULL, changelog_fsync_thread, priv);
+        }
+
+        if (ret)
+                changelog_cleanup_helper_threads (this, priv);
+
+ out:
+        return ret;
+}
+
+/* cleanup the notifier thread */
+static int
+changelog_cleanup_notifier (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret = 0;
+
+        if (priv->cn.notify_th) {
+                changelog_thread_cleanup (this, priv->cn.notify_th);
+                priv->cn.notify_th = 0;
+
+                ret = close (priv->wfd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error closing writer end of notifier pipe"
+                                " (reason: %s)", strerror (errno));
+        }
+
+        return ret;
+}
+
+/* spawn the notifier thread - nop if already running */
+static int
+changelog_spawn_notifier (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret        = 0;
+        int flags      = 0;
+        int pipe_fd[2] = {0, 0};
+
+        if (priv->cn.notify_th)
+                goto out; /* notifier thread already running */
+
+        ret = pipe (pipe_fd);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot create pipe (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        /* writer is non-blocking */
+        flags = fcntl (pipe_fd[1], F_GETFL);
+        flags |= O_NONBLOCK;
+
+        ret = fcntl (pipe_fd[1], F_SETFL, flags);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set O_NONBLOCK flag");
+                goto out;
+        }
+
+        priv->wfd = pipe_fd[1];
+
+        priv->cn.this = this;
+        priv->cn.rfd  = pipe_fd[0];
+
+        ret = gf_thread_create (&priv->cn.notify_th,
+				NULL, changelog_notifier, priv);
+
+ out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_changelog_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        " init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+static int
+changelog_init (xlator_t *this, changelog_priv_t *priv)
+{
+        int                  i   = 0;
+        int                  ret = -1;
+        struct timeval       tv  = {0,};
+        changelog_log_data_t cld = {0,};
+
+        ret = gettimeofday (&tv, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gettimeofday() failure");
+                goto out;
+        }
+
+        priv->slice.tv_start = tv;
+
+        priv->maps[CHANGELOG_TYPE_DATA]     = "D ";
+        priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
+        priv->maps[CHANGELOG_TYPE_ENTRY]    = "E ";
+
+        for (; i < CHANGELOG_MAX_TYPE; i++) {
+                /* start with version 1 */
+                priv->slice.changelog_version[i] = 1;
+        }
+
+        if (!priv->active)
+                return ret;
+
+        /* spawn the notifier thread */
+        ret = changelog_spawn_notifier (this, priv);
+        if (ret)
+                goto out;
+
+        /**
+         * start with a fresh changelog file every time. this is done
+         * in case there was an encoding change. so... things are kept
+         * simple here.
+         */
+        ret = changelog_fill_rollover_data (&cld, _gf_false);
+        if (ret)
+                goto out;
+
+        LOCK (&priv->lock);
+        {
+                ret = changelog_inject_single_event (this, priv, &cld);
+        }
+        UNLOCK (&priv->lock);
+
+        /* ... and finally spawn the helpers threads */
+        ret = changelog_spawn_helper_threads (this, priv);
+
+ out:
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int                     ret            = 0;
+        char                   *tmp            = NULL;
+        changelog_priv_t       *priv           = NULL;
+        gf_boolean_t            active_earlier = _gf_true;
+        gf_boolean_t            active_now     = _gf_true;
+        changelog_time_slice_t *slice          = NULL;
+        changelog_log_data_t    cld            = {0,};
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+
+        ret = -1;
+        active_earlier = priv->active;
+
+        /* first stop the rollover and the fsync thread */
+        changelog_cleanup_helper_threads (this, priv);
+
+        GF_OPTION_RECONF ("changelog-dir", tmp, options, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-dir\" option is not set");
+                goto out;
+        }
+
+        GF_FREE (priv->changelog_dir);
+        priv->changelog_dir = gf_strdup (tmp);
+        if (!priv->changelog_dir)
+                goto out;
+
+        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+        if (ret)
+                goto out;
+
+        GF_OPTION_RECONF ("changelog", active_now, options, bool, out);
+
+        /**
+         * changelog_handle_change() handles changes that could possibly
+         * have been submit changes before changelog deactivation.
+         */
+        if (!active_now)
+                priv->active = _gf_false;
+
+        GF_OPTION_RECONF ("op-mode", tmp, options, str, out);
+        changelog_assign_opmode (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_RECONF ("encoding", tmp, options, str, out);
+        changelog_assign_encoding (priv, tmp);
+
+        GF_OPTION_RECONF ("rollover-time",
+                          priv->rollover_time, options, int32, out);
+        GF_OPTION_RECONF ("fsync-interval",
+                          priv->fsync_interval, options, int32, out);
+
+        if (active_now || active_earlier) {
+                ret = changelog_fill_rollover_data (&cld, !active_now);
+                if (ret)
+                        goto out;
+
+                slice = &priv->slice;
+
+                LOCK (&priv->lock);
+                {
+                        ret = changelog_inject_single_event (this, priv, &cld);
+                        if (!ret && active_now)
+                                SLICE_VERSION_UPDATE (slice);
+                }
+                UNLOCK (&priv->lock);
+
+                if (ret)
+                        goto out;
+
+                if (active_now) {
+                        ret = changelog_spawn_notifier (this, priv);
+                        if (!ret)
+                                ret = changelog_spawn_helper_threads (this,
+                                                                      priv);
+                } else
+                        ret = changelog_cleanup_notifier (this, priv);
+        }
+
+ out:
+        if (ret) {
+                ret = changelog_cleanup_notifier (this, priv);
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "changelog reconfigured");
+                if (active_now)
+                        priv->active = _gf_true;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int               ret  = -1;
+        char             *tmp  = NULL;
+        changelog_priv_t *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO ("changelog", this, out);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "translator needs a single subvolume");
+                goto out;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "dangling volume. please check volfile");
+                goto out;
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        this->local_pool = mem_pool_new (changelog_local_t, 64);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local memory pool");
+                goto out;
+        }
+
+        LOCK_INIT (&priv->lock);
+
+        GF_OPTION_INIT ("changelog-brick", tmp, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-brick\" option is not set");
+                goto out;
+        }
+
+        priv->changelog_brick = gf_strdup (tmp);
+        if (!priv->changelog_brick)
+                goto out;
+        tmp = NULL;
+
+        GF_OPTION_INIT ("changelog-dir", tmp, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-dir\" option is not set");
+                goto out;
+        }
+
+        priv->changelog_dir = gf_strdup (tmp);
+        if (!priv->changelog_dir)
+                goto out;
+        tmp = NULL;
+
+        /**
+         * create the directory even if change-logging would be inactive
+         * so that consumers can _look_ into it (finding nothing...)
+         */
+        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+        if (ret)
+                goto out;
+
+        GF_OPTION_INIT ("changelog", priv->active, bool, out);
+
+        GF_OPTION_INIT ("op-mode", tmp, str, out);
+        changelog_assign_opmode (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_INIT ("encoding", tmp, str, out);
+        changelog_assign_encoding (priv, tmp);
+
+        GF_OPTION_INIT ("rollover-time", priv->rollover_time, int32, out);
+
+        GF_OPTION_INIT ("fsync-interval", priv->fsync_interval, int32, out);
+
+        changelog_encode_change(priv);
+
+        GF_ASSERT (cb_bootstrap[priv->op_mode].mode == priv->op_mode);
+        priv->cb = &cb_bootstrap[priv->op_mode];
+
+        /* ... now bootstrap the logger */
+        ret = priv->cb->ctor (this, &priv->cd);
+        if (ret)
+                goto out;
+
+        priv->changelog_fd = -1;
+        ret = changelog_init (this, priv);
+        if (ret)
+                goto out;
+
+        gf_log (this->name, GF_LOG_DEBUG, "changelog translator loaded");
+
+ out:
+        if (ret) {
+                if (this->local_pool)
+                        mem_pool_destroy (this->local_pool);
+                if (priv->cb) {
+                        ret = priv->cb->dtor (this, &priv->cd);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "error in cleanup during init()");
+                }
+                GF_FREE (priv->changelog_brick);
+                GF_FREE (priv->changelog_dir);
+                GF_FREE (priv);
+                this->private = NULL;
+        } else
+                this->private = priv;
+
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        int               ret  = -1;
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+
+        if (priv) {
+                ret = priv->cb->dtor (this, &priv->cd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error in fini");
+                mem_pool_destroy (this->local_pool);
+                GF_FREE (priv->changelog_brick);
+                GF_FREE (priv->changelog_dir);
+                GF_FREE (priv);
+        }
+
+        this->private = NULL;
+
+        return;
+}
+
+struct xlator_fops fops = {
+        .mknod        = changelog_mknod,
+        .mkdir        = changelog_mkdir,
+        .create       = changelog_create,
+        .symlink      = changelog_symlink,
+        .writev       = changelog_writev,
+        .truncate     = changelog_truncate,
+        .ftruncate    = changelog_ftruncate,
+        .link         = changelog_link,
+        .rename       = changelog_rename,
+        .unlink       = changelog_unlink,
+        .rmdir        = changelog_rmdir,
+        .setattr      = changelog_setattr,
+        .fsetattr     = changelog_fsetattr,
+        .setxattr     = changelog_setxattr,
+        .fsetxattr    = changelog_fsetxattr,
+        .removexattr  = changelog_removexattr,
+        .fremovexattr = changelog_fremovexattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget = changelog_forget,
+};
+
+struct volume_options options[] = {
+        {.key = {"changelog"},
+         .type = GF_OPTION_TYPE_BOOL,
+         .default_value = "off",
+         .description = "enable/disable change-logging"
+        },
+        {.key = {"changelog-brick"},
+         .type = GF_OPTION_TYPE_PATH,
+         .description = "brick path to generate unique socket file name."
+                       " should be the export directory of the volume strictly."
+        },
+        {.key = {"changelog-dir"},
+         .type = GF_OPTION_TYPE_PATH,
+         .description = "directory for the changelog files"
+        },
+        {.key = {"op-mode"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "realtime",
+         .value = {"realtime"},
+         .description = "operation mode - futuristic operation modes"
+        },
+        {.key = {"encoding"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "ascii",
+         .value = {"binary", "ascii"},
+         .description = "encoding type for changelogs"
+        },
+        {.key = {"rollover-time"},
+         .default_value = "60",
+         .type = GF_OPTION_TYPE_TIME,
+         .description = "time to switch to a new changelog file (in seconds)"
+        },
+        {.key = {"fsync-interval"},
+         .type = GF_OPTION_TYPE_TIME,
+         .default_value = "0",
+         .description = "do not open CHANGELOG file with O_SYNC mode."
+                        " instead perform fsync() at specified intervals"
+        },
+        {.key = {NULL}
+        },
+};
diff --git a/xlators/protocol/legacy/lib/Makefile.am b/xlators/features/compress/Makefile.am
index d471a3f92..a985f42a8 100644
--- a/xlators/protocol/legacy/lib/Makefile.am
+++ b/xlators/features/compress/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/compress/src/Makefile.am b/xlators/features/compress/src/Makefile.am
new file mode 100644
index 000000000..0bf757c06
--- /dev/null
+++ b/xlators/features/compress/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = cdc.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = cdc.h cdc-mem-types.h
+
+cdc_la_LDFLAGS = -module -avoid-version $(LIBZ_LIBS)
+
+cdc_la_SOURCES = cdc.c cdc-helper.c
+cdc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+-shared -nostartfiles  $(LIBZ_CFLAGS)
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/compress/src/cdc-helper.c b/xlators/features/compress/src/cdc-helper.c
new file mode 100644
index 000000000..54432ff45
--- /dev/null
+++ b/xlators/features/compress/src/cdc-helper.c
@@ -0,0 +1,547 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#ifdef HAVE_LIB_Z
+/* gzip header looks something like this
+ * (RFC 1950)
+ *
+ * +---+---+---+---+---+---+---+---+---+---+
+ * |ID1|ID2|CM |FLG|     MTIME     |XFL|OS |
+ * +---+---+---+---+---+---+---+---+---+---+
+ *
+ * Data is usually sent without this header i.e
+ * Data sent = <compressed-data> + trailer(8)
+ * The trailer contains the checksum.
+ *
+ * gzip_header is added only during debugging.
+ * Refer to the function cdc_dump_iovec_to_disk
+ */
+static const char gzip_header[10] =
+        {
+                '\037', '\213', Z_DEFLATED, 0,
+                0,      0,      0,          0,
+                0,      GF_CDC_OS_ID
+        };
+
+static int32_t
+cdc_next_iovec (xlator_t *this, cdc_info_t *ci)
+{
+        int ret = -1;
+
+        ci->ncount++;
+        /* check for iovec overflow -- should not happen */
+        if (ci->ncount == MAX_IOVEC) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib output buffer overflow"
+                        " ->ncount (%d) | ->MAX_IOVEC (%d)",
+                        ci->ncount, MAX_IOVEC);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_put_long (unsigned char *string, unsigned long x)
+{
+        string[0] = (unsigned char) (x & 0xff);
+        string[1] = (unsigned char) ((x & 0xff00) >> 8);
+        string[2] = (unsigned char) ((x & 0xff0000) >> 16);
+        string[3] = (unsigned char) ((x & 0xff000000) >> 24);
+}
+
+static unsigned long
+cdc_get_long (unsigned char *buf)
+{
+        return ((unsigned long) buf[0])
+                | (((unsigned long) buf[1]) << 8)
+                | (((unsigned long) buf[2]) << 16)
+                | (((unsigned long) buf[3]) << 24);
+}
+
+static int32_t
+cdc_init_gzip_trailer (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int   ret = -1;
+        char *buf = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        buf = CURR_VEC(ci).iov_base =
+                (char *) GF_CALLOC (1, GF_CDC_VALIDATION_SIZE,
+                                    gf_cdc_mt_gzip_trailer_t);
+
+        if (!CURR_VEC(ci).iov_base)
+                goto out;
+
+        CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
+
+        cdc_put_long ((unsigned char *)&buf[0], ci->crc);
+        cdc_put_long ((unsigned char *)&buf[4], ci->stream.total_in);
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static int32_t
+cdc_alloc_iobuf_and_init_vec (xlator_t *this,
+                              cdc_priv_t *priv, cdc_info_t *ci,
+                              int size)
+{
+        int           ret       = -1;
+        int           alloc_len = 0;
+        struct iobuf *iobuf     = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        alloc_len = size ? size : ci->buffer_size;
+
+        iobuf = iobuf_get2 (this->ctx->iobuf_pool, alloc_len);
+        if (!iobuf)
+                goto out;
+
+        ret = iobref_add (ci->iobref, iobuf);
+        if (ret)
+                goto out;
+
+        /* Initialize this iovec */
+        CURR_VEC(ci).iov_base = iobuf->ptr;
+        CURR_VEC(ci).iov_len  = alloc_len;
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
+{
+        ci->stream.next_out  = (unsigned char *) CURR_VEC(ci).iov_base;
+        ci->stream.avail_out = size ? size : ci->buffer_size;
+}
+
+/* This routine is for testing and debugging only.
+ * Data written = header(10) + <compressed-data> + trailer(8)
+ * So each gzip dump file is at least 18 bytes in size.
+ */
+void
+cdc_dump_iovec_to_disk (xlator_t *this, cdc_info_t *ci, const char *file)
+{
+        int    i      = 0;
+        int    fd     = 0;
+        size_t writen = 0;
+        size_t total_writen = 0;
+
+        fd = open (file, O_WRONLY|O_CREAT|O_TRUNC, 0777 );
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot open file: %s", file);
+                return;
+        }
+
+        writen = write (fd, (char *) gzip_header, 10);
+        total_writen += writen;
+        for (i = 0; i < ci->ncount; i++) {
+                writen = write (fd, (char *) ci->vec[i].iov_base, ci->vec[i].iov_len);
+                total_writen += writen;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                        "dump'd %zu bytes to %s", total_writen, GF_CDC_DEBUG_DUMP_FILE );
+
+        close (fd);
+}
+
+static int32_t
+cdc_flush_libz_buffer (cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
+                       int (*libz_func)(z_streamp, int),
+                       int flush)
+{
+        int32_t ret = Z_OK;
+        int done = 0;
+        unsigned int deflate_len = 0;
+
+        for (;;) {
+                deflate_len = ci->buffer_size - ci->stream.avail_out;
+
+                if (deflate_len != 0) {
+                        CURR_VEC(ci).iov_len = deflate_len;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret) {
+                                ret = Z_MEM_ERROR;
+                                break;
+                        }
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                if (done) {
+                        ci->ncount--;
+                        break;
+                }
+
+                ret = libz_func (&ci->stream, flush);
+
+                if (ret == Z_BUF_ERROR) {
+                        ret = Z_OK;
+                        ci->ncount--;
+                        break;
+                }
+
+                done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
+
+                if (ret != Z_OK && ret != Z_STREAM_END)
+                        break;
+        }
+
+        return ret;
+}
+
+static int32_t
+do_cdc_compress (struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
+                 cdc_info_t *ci)
+{
+        int ret = -1;
+
+        /* Initialize defalte */
+        ret = deflateInit2 (&ci->stream, priv->cdc_level, Z_DEFLATED,
+                            priv->window_size, priv->mem_level,
+                            Z_DEFAULT_STRATEGY);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to init Zlib (retval: %d)", ret);
+                goto out;
+        }
+
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in  = (unsigned char *) vec->iov_base;
+        ci->stream.avail_in = vec->iov_len;
+
+        ci->crc = crc32 (ci->crc, (const Bytef *) vec->iov_base, vec->iov_len);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d",
+                ci->crc, ci->stream.avail_in, ci->buffer_size);
+
+        /* compress !! */
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = deflate (&ci->stream, Z_NO_FLUSH);
+                if (ret != Z_OK)
+                        break;
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_compress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+              dict_t **xdata)
+{
+        int ret = -1;
+        int i   = 0;
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto out;
+
+        if (!*xdata) {
+                *xdata = dict_new ();
+                if (!*xdata) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot allocate xdata"
+                                " dict");
+                        goto out;
+                }
+        }
+
+        /* data */
+        for (i = 0; i < ci->count; i++) {
+                ret = do_cdc_compress (&ci->vector[i], this, priv, ci);
+                if (ret != Z_OK)
+                        goto deflate_cleanup_out;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, deflate, Z_FINISH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Compression Error: ret (%d)", ret);
+                ret = -1;
+                goto deflate_cleanup_out;
+        }
+
+        /* trailer */
+        ret = cdc_init_gzip_trailer (this, priv, ci);
+        if (ret)
+                goto deflate_cleanup_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Compressed %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+        ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
+
+        /* set deflated canary value for identification */
+        ret = dict_set_int32 (*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
+        if (ret) {
+                /* Send uncompressed data if we can't _tell_ the client
+                 * that deflated data is on it's way. So, we just log
+                 * the faliure and continue as usual.
+                 */
+                 gf_log (this->name, GF_LOG_ERROR,
+                 "Data deflated, but could not set canary"
+                 " value in dict for identification");
+        }
+
+        /* This is to be used in testing */
+        if ( priv->debug ) {
+                cdc_dump_iovec_to_disk (this, ci, GF_CDC_DEBUG_DUMP_FILE );
+        }
+
+ deflate_cleanup_out:
+        (void) deflateEnd(&ci->stream);
+
+ out:
+        return ret;
+}
+
+
+/* deflate content is checked by the presence of a canary
+ * value in the dict as the key
+ */
+static int32_t
+cdc_check_content_for_deflate (dict_t *xdata)
+{
+        return dict_get (xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
+}
+
+static unsigned long
+cdc_extract_crc (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[0]);
+}
+
+static unsigned long
+cdc_extract_size (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[4]);
+}
+
+static int32_t
+cdc_validate_inflate (cdc_info_t *ci, unsigned long crc,
+                      unsigned long len)
+{
+        return !((crc == ci->crc)
+                 /* inflated length is hidden inside
+                  * Zlib stream struct */
+                 && (len == ci->stream.total_out));
+}
+
+static int32_t
+do_cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int            ret          = -1;
+        int            i            = 0;
+        int            len          = 0;
+        char          *inflte       = NULL;
+        char          *trailer      = NULL;
+        struct iovec   vec          = {0,};
+        unsigned long  computed_crc = 0;
+        unsigned long  computed_len = 0;
+
+        ret = inflateInit2 (&ci->stream, priv->window_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib: Unable to initialize inflate");
+                goto out;
+        }
+
+        vec = THIS_VEC(ci, 0);
+
+        trailer = (char *) (((char *) vec.iov_base) + vec.iov_len
+                - GF_CDC_VALIDATION_SIZE);
+
+        /* CRC of uncompressed data */
+        computed_crc = cdc_extract_crc (trailer);
+
+        /* size of uncomrpessed data */
+        computed_len = cdc_extract_size (trailer);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
+                computed_crc, computed_len, ci->buffer_size);
+
+        inflte = vec.iov_base ;
+        len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
+
+        /* allocate buffer of the original length of the data */
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in = (unsigned char *) inflte;
+        ci->stream.avail_in = len;
+
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = inflate (&ci->stream, Z_NO_FLUSH);
+                if (ret == Z_STREAM_ERROR)
+                        break;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, inflate, Z_SYNC_FLUSH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Decompression Error: ret (%d)", ret);
+                ret = -1;
+                goto out;
+        }
+
+        /* compute CRC of the uncompresses data to check for
+         * correctness */
+
+        for (i = 0; i < ci->ncount; i++) {
+                ci->crc = crc32 (ci->crc,
+                                 (const Bytef *) ci->vec[i].iov_base,
+                                 ci->vec[i].iov_len);
+        }
+
+        /* validate inflated data */
+        ret = cdc_validate_inflate (ci, computed_crc, computed_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Checksum or length mismatched in inflated data");
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+                dict_t *xdata)
+{
+        int32_t ret = -1;
+
+        /* check for deflate content */
+        if (!cdc_check_content_for_deflate (xdata)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Content not deflated, passing through ...");
+                goto passthrough_out;
+        }
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto passthrough_out;
+
+        /* do we need to do this? can we assume that one iovec
+         * will hold per request data everytime?
+         *
+         * server/client protocol seems to deal with a single
+         * iovec even if op_ret > 1M. So, it looks ok to
+         * assume that a single iovec will contain all the
+         * data (This saves us a lot from finding the trailer
+         * and the data since it could have been split-up onto
+         * two adjacent iovec's.
+         *
+         * But, in case this translator is loaded above quick-read
+         * for some reason, then it's entirely possible that we get
+         * multiple iovec's...
+         *
+         * This case (handled below) is not tested. (by loading the
+         * xlator below quick-read)
+         */
+
+        /* @@ I_HOPE_THIS_IS_NEVER_HIT */
+        if (ci->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING, "unable to handle"
+                        " multiple iovecs (%d in number)", ci->count);
+                goto inflate_cleanup_out;
+                /* TODO: coallate all iovecs in one */
+        }
+
+        ret = do_cdc_decompress (this, priv, ci);
+        if (ret)
+                goto inflate_cleanup_out;
+
+        ci->nbytes = ci->stream.total_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Inflated %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+ inflate_cleanup_out:
+        (void) inflateEnd (&ci->stream);
+
+ passthrough_out:
+        return ret;
+}
+
+#endif
diff --git a/xlators/features/compress/src/cdc-mem-types.h b/xlators/features/compress/src/cdc-mem-types.h
new file mode 100644
index 000000000..ead2c70ba
--- /dev/null
+++ b/xlators/features/compress/src/cdc-mem-types.h
@@ -0,0 +1,23 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_MEM_TYPES_H
+#define __CDC_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_cdc_mem_types {
+        gf_cdc_mt_priv_t         = gf_common_mt_end + 1,
+        gf_cdc_mt_vec_t          = gf_common_mt_end + 2,
+        gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
+        gf_cdc_mt_end            = gf_common_mt_end + 4,
+};
+
+#endif
diff --git a/xlators/features/compress/src/cdc.c b/xlators/features/compress/src/cdc.c
new file mode 100644
index 000000000..67fc52505
--- /dev/null
+++ b/xlators/features/compress/src/cdc.c
@@ -0,0 +1,361 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+static void
+cdc_cleanup_iobref (cdc_info_t *ci)
+{
+        assert(ci->iobref != NULL);
+        iobref_clear (ci->iobref);
+}
+
+int32_t
+cdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iovec *vector, int32_t count,
+               struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
+{
+        int         ret   = -1;
+        cdc_priv_t *priv  = NULL;
+        cdc_info_t  ci    = {0,};
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+        priv = this->private;
+
+        if (op_ret <= 0)
+                goto default_out;
+
+        if ( (priv->min_file_size != 0)
+             && (op_ret < priv->min_file_size) )
+                goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = op_ret;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A readv compresses on the server side and decompresses on the client side
+ */
+        if (priv->op_mode == GF_CDC_MODE_SERVER) {
+                ret = cdc_compress (this, priv, &ci, &xdata);
+        } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+                ret = cdc_decompress (this, priv, &ci, xdata);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid operation mode (%d)", priv->op_mode);
+        }
+
+        if (ret)
+                goto default_out;
+
+        STACK_UNWIND_STRICT (readv, frame, ci.nbytes, op_errno,
+                             ci.vec, ci.ncount, stbuf, iobref,
+                             xdata);
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+cdc_readv (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, size_t size, off_t offset, uint32_t flags,
+           dict_t *xdata)
+{
+        fop_readv_cbk_t cbk = NULL;
+
+#ifdef HAVE_LIB_Z
+        cbk = cdc_readv_cbk;
+#else
+        cbk = default_readv_cbk;
+#endif
+        STACK_WIND (frame, cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+        return 0;
+}
+
+int32_t
+cdc_writev_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+cdc_writev (call_frame_t *frame,
+            xlator_t *this,
+            fd_t *fd,
+            struct iovec *vector,
+            int32_t count,
+            off_t offset,
+            uint32_t flags,
+            struct iobref *iobref, dict_t *xdata)
+{
+        int          ret   = -1;
+        cdc_priv_t  *priv  = NULL;
+        cdc_info_t   ci    = {0,};
+        size_t       isize = 0;
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+        priv = this->private;
+
+        isize =  iov_length(vector, count);
+
+        if (isize <= 0)
+            goto default_out;
+
+        if ( (priv->min_file_size != 0) && (isize < priv->min_file_size) )
+            goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = isize;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A writev compresses on the client side and decompresses on the server side
+ */
+	    if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+		    ret = cdc_compress (this, priv, &ci, &xdata);
+	    } else if (priv->op_mode == GF_CDC_MODE_SERVER) {
+		    ret = cdc_decompress (this, priv, &ci, xdata);
+	    } else {
+		    gf_log (this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ", priv->op_mode);
+	    }
+
+	    if (ret)
+		    goto default_out;
+
+	    STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, ci.vec, ci.ncount, offset, flags,
+                    iobref, xdata);
+
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+        STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, vector, count, offset, flags,
+                    iobref, xdata);
+        return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_cdc_mt_end);
+
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                       "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int         ret      = -1;
+        char       *temp_str = NULL;
+        cdc_priv_t *priv     = NULL;
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, err);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Need subvolume == 1");
+                goto err;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Dangling volume. Check volfile");
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_cdc_mt_priv_t);
+        if (!priv) {
+                goto err;
+        }
+
+        /* Check if debug mode is turned on */
+        GF_OPTION_INIT ("debug", priv->debug, bool, err);
+        if( priv->debug ) {
+                gf_log (this->name, GF_LOG_DEBUG, "CDC debug option turned on");
+        }
+
+        /* Set Gzip Window Size */
+        GF_OPTION_INIT ("window-size", priv->window_size, int32, err);
+        if ( (priv->window_size > GF_CDC_MAX_WINDOWSIZE)
+             || (priv->window_size < GF_CDC_DEF_WINDOWSIZE) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip window size (%d), using default",
+                        priv->window_size);
+                priv->window_size = GF_CDC_DEF_WINDOWSIZE;
+        }
+
+        /* Set Gzip (De)Compression Level */
+        GF_OPTION_INIT ("compression-level", priv->cdc_level, int32, err);
+        if ( ((priv->cdc_level < 1) || (priv->cdc_level > 9))
+             && (priv->cdc_level != GF_CDC_DEF_COMPRESSION) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip (de)compression level (%d),"
+                        " using default", priv->cdc_level);
+                priv->cdc_level = GF_CDC_DEF_COMPRESSION;
+        }
+
+        /* Set Gzip Memory Level */
+        GF_OPTION_INIT ("mem-level", priv->mem_level, int32, err);
+        if ( (priv->mem_level < 1) || (priv->mem_level > 9) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip memory level, using the default");
+                priv->mem_level = GF_CDC_DEF_MEMLEVEL;
+        }
+
+        /* Set min file size to enable compression */
+        GF_OPTION_INIT ("min-size", priv->min_file_size, int32, err);
+
+        /* Mode of operation - Server/Client */
+        ret = dict_get_str (this->options, "mode", &temp_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Operation mode not specified !!");
+                goto err;
+        }
+
+        if (GF_CDC_MODE_IS_CLIENT (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_CLIENT;
+        } else if (GF_CDC_MODE_IS_SERVER (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_SERVER;
+        } else {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Bogus operation mode (%s) specified", temp_str);
+                goto err;
+        }
+
+        this->private = priv;
+        gf_log (this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",temp_str);
+        return 0;
+
+ err:
+        if (priv)
+                GF_FREE (priv);
+
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        cdc_priv_t *priv = this->private;
+
+        if (priv)
+                GF_FREE (priv);
+        this->private = NULL;
+        return;
+}
+
+struct xlator_fops fops = {
+        .readv  = cdc_readv,
+        .writev = cdc_writev,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {"window-size"},
+          .default_value = "-15",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Size of the zlib history buffer."
+        },
+        { .key  = {"mem-level"},
+          .default_value = "8",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Memory allocated for internal compression state. "
+                         "1 uses minimum memory but is slow and reduces "
+                         "compression ratio; memLevel=9 uses maximum memory "
+                         "for optimal speed. The default value is 8."
+        },
+        { .key  = {"compression-level"},
+          .default_value = "-1",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Compression levels \n"
+                         "0 : no compression, 1 : best speed, \n"
+                         "9 : best compression, -1 : default compression "
+        },
+        { .key  = {"min-size"},
+          .default_value = "0",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Data is compressed only when its size exceeds this."
+        },
+        { .key  = {"mode"},
+          .value = {"server", "client"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Set on the basis of where the xlator is loaded. "
+                         "This option should NOT be configured by user."
+        },
+        { .key = {"debug"},
+          .default_value = "false",
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "This is used in testing. Will dump compressed data "
+                         "to disk as a gzip file."
+        },
+        { .key  = {NULL}
+        },
+};
diff --git a/xlators/features/compress/src/cdc.h b/xlators/features/compress/src/cdc.h
new file mode 100644
index 000000000..71f4d2317
--- /dev/null
+++ b/xlators/features/compress/src/cdc.h
@@ -0,0 +1,107 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_H
+#define __CDC_H
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#include "xlator.h"
+
+#ifndef MAX_IOVEC
+#define MAX_IOVEC 16
+#endif
+
+typedef struct cdc_priv {
+        int window_size;
+        int mem_level;
+        int cdc_level;
+        int min_file_size;
+        int op_mode;
+        gf_boolean_t debug;
+        gf_lock_t lock;
+} cdc_priv_t;
+
+typedef struct cdc_info {
+        /* input bits */
+        int            count;
+        int32_t        ibytes;
+        struct iovec  *vector;
+        struct iatt   *buf;
+
+        /* output bits */
+        int            ncount;
+        int            nbytes;
+        int            buffer_size;
+        struct iovec   vec[MAX_IOVEC];
+        struct iobref *iobref;
+
+        /* zlib bits */
+#ifdef HAVE_LIB_Z
+        z_stream      stream;
+#endif
+        unsigned long crc;
+} cdc_info_t;
+
+#define NVEC(ci) (ci->ncount - 1)
+#define CURR_VEC(ci) ci->vec[ci->ncount - 1]
+#define THIS_VEC(ci, i) ci->vector[i]
+
+/* Gzip defaults */
+#define GF_CDC_DEF_WINDOWSIZE  -15 /* default value */
+#define GF_CDC_MAX_WINDOWSIZE  -8  /* max value     */
+
+#ifdef HAVE_LIB_Z
+#define GF_CDC_DEF_COMPRESSION Z_DEFAULT_COMPRESSION
+#else
+#define GF_CDC_DEF_COMPRESSION -1
+#endif
+
+#define GF_CDC_DEF_MEMLEVEL    8
+#define GF_CDC_DEF_BUFFERSIZE  262144 // 256K - default compression buffer size
+
+/* Operation mode
+ * If xlator is loaded on client, readv decompresses and writev compresses
+ * If xlator is loaded on server, readv compresses and writev decompresses
+ */
+#define GF_CDC_MODE_CLIENT   0
+#define GF_CDC_MODE_SERVER   1
+
+/* min size of data to do cmpression
+ * 0 == compress even 1byte
+ */
+#define GF_CDC_MIN_CHUNK_SIZE 0
+
+#define GF_CDC_VALIDATION_SIZE 8
+
+#define GF_CDC_OS_ID 0xFF
+#define GF_CDC_DEFLATE_CANARY_VAL "deflate"
+#define GF_CDC_DEBUG_DUMP_FILE "/tmp/cdcdump.gz"
+
+#define GF_CDC_MODE_IS_CLIENT(m) \
+        (strcmp (m, "client") == 0)
+
+#define GF_CDC_MODE_IS_SERVER(m) \
+        (strcmp (m, "server") == 0)
+
+int32_t
+cdc_compress (xlator_t *this,
+              cdc_priv_t *priv,
+              cdc_info_t *ci,
+              dict_t **xdata);
+int32_t
+cdc_decompress (xlator_t *this,
+                cdc_priv_t *priv,
+                cdc_info_t *ci,
+                dict_t *xdata);
+
+#endif
diff --git a/xlators/features/filter/src/Makefile.am b/xlators/features/filter/src/Makefile.am
index d473b9ea1..d1fda8b0a 100644
--- a/xlators/features/filter/src/Makefile.am
+++ b/xlators/features/filter/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = filter.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-filter_la_LDFLAGS = -module -avoidversion
+filter_la_LDFLAGS = -module -avoid-version
 
 filter_la_SOURCES = filter.c
 filter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = filter-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/filter/src/filter-mem-types.h b/xlators/features/filter/src/filter-mem-types.h
index de2cb9665..47a17249b 100644
--- a/xlators/features/filter/src/filter-mem-types.h
+++ b/xlators/features/filter/src/filter-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __FILTER_MEM_TYPES_H__
 #define __FILTER_MEM_TYPES_H__
 
diff --git a/xlators/features/filter/src/filter.c b/xlators/features/filter/src/filter.c
index eda042f35..1d4887b71 100644
--- a/xlators/features/filter/src/filter.c
+++ b/xlators/features/filter/src/filter.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/bindings/python/Makefile.am b/xlators/features/gfid-access/Makefile.am
index af437a64d..af437a64d 100644
--- a/xlators/bindings/python/Makefile.am
+++ b/xlators/features/gfid-access/Makefile.am
diff --git a/xlators/features/gfid-access/src/Makefile.am b/xlators/features/gfid-access/src/Makefile.am
new file mode 100644
index 000000000..db53affaa
--- /dev/null
+++ b/xlators/features/gfid-access/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = gfid-access.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+gfid_access_la_LDFLAGS = -module -avoid-version
+
+gfid_access_la_SOURCES = gfid-access.c
+gfid_access_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = gfid-access.h gfid-access-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/gfid-access/src/gfid-access-mem-types.h b/xlators/features/gfid-access/src/gfid-access-mem-types.h
new file mode 100644
index 000000000..168d67b43
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access-mem-types.h
@@ -0,0 +1,23 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GFID_ACCESS_MEM_TYPES_H
+#define _GFID_ACCESS_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_changelog_mem_types {
+        gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
+        gf_gfid_access_mt_gfid_t,
+        gf_gfid_access_mt_end
+};
+
+#endif
+
diff --git a/xlators/features/gfid-access/src/gfid-access.c b/xlators/features/gfid-access/src/gfid-access.c
new file mode 100644
index 000000000..8e614397c
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access.c
@@ -0,0 +1,1287 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "gfid-access.h"
+#include "inode.h"
+#include "byte-order.h"
+
+
+
+void
+ga_newfile_args_free (ga_newfile_args_t *args)
+{
+        if (!args)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        if (S_ISLNK (args->st_mode) && args->args.symlink.linkpath) {
+                GF_FREE (args->args.symlink.linkpath);
+                args->args.symlink.linkpath = NULL;
+        }
+
+        mem_put (args);
+out:
+        return;
+}
+
+
+void
+ga_heal_args_free (ga_heal_args_t *args)
+{
+        if (!args)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        mem_put (args);
+out:
+        return;
+}
+
+
+ga_newfile_args_t *
+ga_newfile_parse_args (xlator_t *this, data_t *data)
+{
+        ga_newfile_args_t *args     = NULL;
+        ga_private_t      *priv     = NULL;
+        int                len      = 0;
+        int                blob_len = 0;
+        int                min_len  = 0;
+        void              *blob     = NULL;
+
+        priv = this->private;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
+                + sizeof (args->st_mode) + 2 + 2;
+        if (blob_len < min_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid length: Total length is less "
+                        "than minimum length.");
+                goto err;
+        }
+
+        args = mem_get0 (priv->newfile_args_pool);
+        if (args == NULL)
+                goto err;
+
+        args->uid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        args->gid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        args->st_mode = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+        if (len == blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. No null byte present.",
+                        args->gfid);
+                goto err;
+        }
+
+        args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
+        if (args->bname == NULL)
+                goto err;
+
+        memcpy (args->bname, blob, (len + 1));
+        blob += (len + 1);
+        blob_len -= (len + 1);
+
+        if (S_ISDIR (args->st_mode)) {
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+                if (blob_len < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+        } else if (S_ISLNK (args->st_mode)) {
+                len = strnlen (blob, blob_len);
+                if (len == blob_len) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
+                                                         gf_common_mt_char);
+                if (args->args.symlink.linkpath == NULL)
+                        goto err;
+
+                memcpy (args->args.symlink.linkpath, blob, (len + 1));
+                blob += (len + 1);
+                blob_len -= (len + 1);
+        } else {
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+        }
+
+        if (blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. Invalid length",
+                        args->gfid);
+                goto err;
+        }
+
+        return args;
+
+err:
+        if (args)
+                ga_newfile_args_free (args);
+
+        return NULL;
+}
+
+ga_heal_args_t *
+ga_heal_parse_args (xlator_t *this, data_t *data)
+{
+        ga_heal_args_t *args     = NULL;
+        ga_private_t   *priv     = NULL;
+        void           *blob     = NULL;
+        int             len      = 0;
+        int             blob_len = 0;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        priv = this->private;
+
+        /* bname should at least contain a character */
+        if (blob_len < (sizeof (args->gfid) + 2))
+                goto err;
+
+        args = mem_get0 (priv->heal_args_pool);
+        if (!args)
+                goto err;
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+                goto err;
+
+        args->bname = GF_CALLOC (1, len + 1, gf_common_mt_char);
+        if (!args->bname)
+                goto err;
+
+        memcpy (args->bname, blob, len);
+        blob_len -= (len + 1);
+
+        if (blob_len)
+                goto err;
+
+        return args;
+
+err:
+        if (args)
+                ga_heal_args_free (args);
+
+        return NULL;
+}
+
+static int32_t
+ga_fill_tmp_loc (loc_t *loc, xlator_t *this, uuid_t gfid,
+                 char *bname, dict_t *xdata, loc_t *new_loc)
+{
+        int       ret    = -1;
+        uint64_t  value  = 0;
+        inode_t  *parent = NULL;
+
+        parent = loc->inode;
+        ret = inode_ctx_get (loc->inode, this, &value);
+        if (!ret) {
+                parent = (void *)value;
+                if (uuid_is_null (parent->gfid))
+                        parent = loc->inode;
+        }
+
+        /* parent itself should be looked up */
+        uuid_copy (new_loc->pargfid, parent->gfid);
+        new_loc->parent = inode_ref (parent);
+
+        new_loc->inode = inode_grep (parent->table, parent, bname);
+        if (!new_loc->inode)
+                new_loc->inode = inode_new (parent->table);
+
+        loc_path (new_loc, bname);
+        new_loc->name = basename (new_loc->path);
+
+        /* As GFID would not be set on the entry yet, lets not send entry
+           gfid in the request */
+        /*uuid_copy (new_loc->gfid, (const unsigned char *)gfid); */
+
+        ret = dict_set_static_bin (xdata, "gfid-req", gfid, 16);
+        if (ret < 0)
+                goto out;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+
+static gf_boolean_t
+__is_gfid_access_dir (uuid_t gfid)
+{
+        uuid_t  aux_gfid;
+
+        memset (aux_gfid, 0, 16);
+        aux_gfid[15] = GF_AUX_GFID;
+
+        if (uuid_compare (gfid, aux_gfid) == 0)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+int32_t
+ga_forget (xlator_t *this, inode_t *inode)
+{
+        int       ret = -1;
+        uint64_t  value = 0;
+        inode_t  *tmp_inode = NULL;
+
+        ret = inode_ctx_del (inode, this, &value);
+        if (ret)
+                goto out;
+
+        tmp_inode = (void *)value;
+        inode_unref (tmp_inode);
+
+out:
+        return 0;
+}
+
+
+static int
+ga_heal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno,
+             inode_t *inode, struct iatt *stat, dict_t *dict,
+             struct iatt *postparent)
+{
+        call_frame_t *orig_frame = NULL;
+
+        orig_frame = frame->local;
+        frame->local = NULL;
+
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, dict);
+
+        return 0;
+}
+
+static int32_t
+ga_newentry_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                         struct iatt *statpost,
+                         dict_t *xdata)
+{
+        ga_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret,
+                             op_errno, xdata);
+
+        loc_wipe (&local->loc);
+        mem_put (local);
+
+        return 0;
+}
+
+static int
+ga_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata)
+{
+        ga_local_t *local = NULL;
+        struct iatt temp_stat = {0,};
+
+        local = frame->local;
+
+        if (!local->uid && !local->gid)
+                goto done;
+
+        temp_stat.ia_uid = local->uid;
+        temp_stat.ia_gid = local->gid;
+
+        STACK_WIND (frame, ga_newentry_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, &local->loc, &temp_stat,
+                    (GF_SET_ATTR_UID | GF_SET_ATTR_GID), xdata);
+
+        return 0;
+
+done:
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret,
+                             op_errno, xdata);
+
+        loc_wipe (&local->loc);
+        mem_put (local);
+
+        return 0;
+}
+
+int32_t
+ga_new_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+              dict_t *xdata)
+{
+        int                ret       = -1;
+        ga_newfile_args_t *args      = NULL;
+        loc_t              tmp_loc   = {0,};
+        call_frame_t      *new_frame = NULL;
+        mode_t             mode      = 0;
+        ga_local_t        *local     = NULL;
+        uuid_t             gfid      = {0,};
+
+        args = ga_newfile_parse_args (this, data);
+        if (!args)
+                goto out;
+
+        ret = uuid_parse (args->gfid, gfid);
+        if (ret)
+                goto out;
+
+        if (!xdata)
+                xdata = dict_new ();
+
+        ret = ga_fill_tmp_loc (loc, this, gfid,
+                               args->bname, xdata, &tmp_loc);
+        if (ret)
+                goto out;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+
+        local = mem_get0 (this->local_pool);
+        local->orig_frame = frame;
+
+        local->uid = args->uid;
+        local->gid = args->gid;
+
+        loc_copy (&local->loc, &tmp_loc);
+
+        new_frame->local = local;
+
+        if (S_ISDIR (args->st_mode)) {
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                            &tmp_loc, args->args.mkdir.mode,
+                            args->args.mkdir.umask, xdata);
+        } else if (S_ISLNK (args->st_mode)) {
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                            args->args.symlink.linkpath,
+                            &tmp_loc, 0, xdata);
+        } else {
+                /* use 07777 (4 7s) for considering the Sticky bits etc) */
+                mode = (S_IFMT & args->st_mode) |
+                        (07777 & args->args.mknod.mode);;
+
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                            &tmp_loc, mode,
+                            args->args.mknod.rdev, args->args.mknod.umask,
+                            xdata);
+        }
+
+        ret = 0;
+out:
+        ga_newfile_args_free (args);
+
+        return ret;
+}
+
+int32_t
+ga_heal_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+               dict_t *xdata)
+{
+        int             ret       = -1;
+        ga_heal_args_t *args      = NULL;
+        loc_t           tmp_loc   = {0,};
+        call_frame_t   *new_frame = NULL;
+        uuid_t          gfid      = {0,};
+
+        args = ga_heal_parse_args (this, data);
+        if (!args)
+                goto out;
+
+        ret = uuid_parse (args->gfid, gfid);
+        if (ret)
+                goto out;
+
+        if (!xdata)
+                xdata = dict_new ();
+
+        ret = ga_fill_tmp_loc (loc, this, gfid, args->bname,
+                               xdata, &tmp_loc);
+        if (ret)
+                goto out;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+        new_frame->local = (void *)frame;
+
+        STACK_WIND (new_frame, ga_heal_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    &tmp_loc, xdata);
+
+        ret = 0;
+out:
+        if (args)
+                ga_heal_args_free (args);
+
+        return ret;
+}
+
+int32_t
+ga_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+ga_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+
+        data_t  *data     = NULL;
+        int      op_errno = ENOMEM;
+        int      ret      = 0;
+        inode_t *unref    = NULL;
+
+        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
+            ((loc->parent &&
+              __is_root_gfid (loc->parent->gfid)) ||
+             __is_root_gfid (loc->pargfid))) {
+                op_errno = EPERM;
+                goto err;
+        }
+
+        data = dict_get (dict, GF_FUSE_AUX_GFID_NEWFILE);
+        if (data) {
+                ret = ga_new_entry (frame, this, loc, data, xdata);
+                if (ret)
+                        goto err;
+                return 0;
+        }
+
+        data = dict_get (dict, GF_FUSE_AUX_GFID_HEAL);
+        if (data) {
+                ret = ga_heal_entry (frame, this, loc, data, xdata);
+                if (ret)
+                        goto err;
+                return 0;
+        }
+
+        //If the inode is a virtual inode change the inode otherwise perform
+        //the operation on same inode
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, ga_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+        return 0;
+}
+
+
+int32_t
+ga_virtual_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        int       j           = 0;
+        int       i           = 0;
+        int       ret         = 0;
+        uint64_t  temp_ino    = 0;
+        inode_t  *cbk_inode   = NULL;
+        inode_t  *true_inode  = NULL;
+        uuid_t    random_gfid = {0,};
+
+        if (frame->local)
+                cbk_inode = frame->local;
+        else
+                cbk_inode = inode;
+
+        frame->local = NULL;
+        if (op_ret)
+                goto unwind;
+
+        if (!IA_ISDIR (buf->ia_type))
+                goto unwind;
+
+        /* need to send back a different inode for linking in itable */
+        if (cbk_inode == inode) {
+                /* check if the inode is in the 'itable' or
+                   if its just previously discover()'d inode */
+                true_inode = inode_find (inode->table, buf->ia_gfid);
+                if (!true_inode) {
+                        cbk_inode = inode_new (inode->table);
+
+                        if (!cbk_inode) {
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                                goto unwind;
+                        }
+                        /* the inode is not present in itable, ie, the actual
+                           path is not yet looked up. Use the current inode
+                           itself for now */
+
+                        inode_link (inode, NULL, NULL, buf);
+                } else {
+                        /* 'inode_ref()' has been done in inode_find() */
+                        inode = true_inode;
+                }
+
+                ret = inode_ctx_put (cbk_inode, this, (uint64_t)inode);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set the inode ctx with"
+                                "the actual inode");
+                        if (inode)
+                                inode_unref (inode);
+                }
+                inode = NULL;
+        }
+
+        if (!uuid_is_null (cbk_inode->gfid)) {
+                /* if the previous linked inode is used, use the
+                   same gfid */
+                uuid_copy (random_gfid, cbk_inode->gfid);
+        } else {
+                /* replace the buf->ia_gfid to a random gfid
+                   for directory, for files, what we received is fine */
+                uuid_generate (random_gfid);
+        }
+
+        uuid_copy (buf->ia_gfid, random_gfid);
+
+        for (i = 15; i > (15 - 8); i--) {
+                temp_ino += (uint64_t)(buf->ia_gfid[i]) << j;
+                j += 8;
+        }
+        buf->ia_ino = temp_ino;
+
+unwind:
+        /* Lookup on non-existing gfid returns ESTALE.
+           Convert into ENOENT for virtual lookup*/
+        if (op_errno == ESTALE)
+               op_errno = ENOENT;
+
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, cbk_inode, buf,
+                             xdata, postparent);
+
+        return 0;
+}
+
+int32_t
+ga_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        ga_private_t *priv = NULL;
+
+        /* if the entry in question is not 'root',
+           then follow the normal path */
+        if (op_ret || !__is_root_gfid(buf->ia_gfid))
+                goto unwind;
+
+        priv = this->private;
+
+        /* do we need to copy root stbuf everytime? */
+        /* mostly yes, as we want to have the 'stat' info show latest
+           in every _cbk() */
+
+        /* keep the reference for root stat buf */
+        priv->root_stbuf = *buf;
+        priv->gfiddir_stbuf = priv->root_stbuf;
+        priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
+        priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
+
+unwind:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
+        return 0;
+}
+
+int32_t
+ga_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        ga_private_t *priv     = NULL;
+        int           ret      = -1;
+        uuid_t        tmp_gfid = {0,};
+        loc_t         tmp_loc  = {0,};
+        uint64_t      value    = 0;
+        inode_t      *inode    = NULL;
+        inode_t      *true_inode    = NULL;
+        int32_t       op_errno = ENOENT;
+
+        /* if its discover(), no need for any action here */
+        if (!loc->name)
+                goto wind;
+
+        /* if its revalidate, and inode is not of type directory,
+           proceed with 'wind' */
+        if (loc->inode && loc->inode->ia_type &&
+            !IA_ISDIR (loc->inode->ia_type)) {
+
+                /* a revalidate on ".gfid/<dentry>" is possible, check for it */
+                if (((loc->parent &&
+                      __is_gfid_access_dir (loc->parent->gfid)) ||
+                     __is_gfid_access_dir (loc->pargfid))) {
+
+                        /* here, just send 'loc->gfid' and 'loc->inode' */
+                        tmp_loc.inode = inode_ref (loc->inode);
+                        uuid_copy (tmp_loc.gfid, loc->inode->gfid);
+
+                        STACK_WIND (frame, default_lookup_cbk,
+                                    FIRST_CHILD(this),
+                                    FIRST_CHILD(this)->fops->lookup,
+                                    &tmp_loc, xdata);
+
+                        inode_unref (tmp_loc.inode);
+
+                        return 0;
+                }
+
+                /* not something to bother, continue the flow */
+                goto wind;
+        }
+
+        priv = this->private;
+
+        /* need to check if the lookup is on virtual dir */
+        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
+            ((loc->parent && __is_root_gfid (loc->parent->gfid)) ||
+             __is_root_gfid (loc->pargfid))) {
+                /* this means, the query is on '/.gfid', return the fake stat,
+                   and say success */
+
+                STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc->inode,
+                                     &priv->gfiddir_stbuf, xdata,
+                                     &priv->root_stbuf);
+                return 0;
+        }
+
+        /* now, check if the lookup() is on an existing entry,
+           but on gfid-path */
+        if (!((loc->parent && __is_gfid_access_dir (loc->parent->gfid)) ||
+              __is_gfid_access_dir (loc->pargfid))) {
+                if (!loc->parent)
+                        goto wind;
+
+                ret = inode_ctx_get (loc->parent, this, &value);
+                if (ret)
+                        goto wind;
+
+                inode = (inode_t *) value;
+
+                ret = loc_copy_overload_parent (&tmp_loc, loc, inode);
+                if (ret)
+                        goto err;
+
+                STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->lookup, &tmp_loc, xdata);
+
+                loc_wipe (&tmp_loc);
+                return 0;
+        }
+
+        /* make sure the 'basename' is actually a 'canonical-gfid',
+           otherwise, return error */
+        ret = uuid_parse (loc->name, tmp_gfid);
+        if (ret)
+                goto err;
+
+        /* if its fresh lookup, go ahead and send it down, if not,
+           for directory, we need indirection to actual dir inode */
+        if (!(loc->inode && loc->inode->ia_type))
+                goto discover;
+
+        /* revalidate on directory */
+        ret = inode_ctx_get (loc->inode, this, &value);
+        if (ret)
+                goto err;
+
+        inode = (void *)value;
+
+        /* valid inode, already looked up, work on that */
+        if (inode->ia_type)
+                goto discover;
+
+        /* check if the inode is in the 'itable' or
+           if its just previously discover()'d inode */
+        true_inode = inode_find (loc->inode->table, tmp_gfid);
+        if (true_inode) {
+                /* time do another lookup and update the context
+                   with proper inode */
+                op_errno = ESTALE;
+                goto err;
+        }
+
+discover:
+        /* for the virtual entries, we don't need to send 'gfid-req' key, as
+           for these entries, we don't want to 'set' a new gfid */
+        if (xdata)
+                dict_del (xdata, "gfid-req");
+
+        uuid_copy (tmp_loc.gfid, tmp_gfid);
+
+        /* if revalidate, then we need to have the proper reference */
+        if (inode) {
+                tmp_loc.inode = inode_ref (inode);
+                frame->local = loc->inode;
+        } else {
+                tmp_loc.inode = inode_ref (loc->inode);
+        }
+
+        STACK_WIND (frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+
+        inode_unref (tmp_loc.inode);
+
+        return 0;
+
+wind:
+        /* used for all the normal lookup path */
+        STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, loc->inode,
+                             &priv->gfiddir_stbuf, xdata,
+                             &priv->root_stbuf);
+        return 0;
+}
+
+int
+ga_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                    xdata);
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, loc->inode,
+                             NULL, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int
+ga_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_create_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, xdata);
+
+        return 0;
+
+}
+
+int
+ga_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_symlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev,
+                    umask, xdata);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+          dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_rmdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flag, xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL,
+                             NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+           dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_unlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL,
+                             NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_rename (call_frame_t *frame, xlator_t *this,
+           loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *oldloc_unref = NULL;
+        inode_t *newloc_unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
+        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, oldloc, oldloc_unref,
+                                         handle_newloc);
+
+handle_newloc:
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, newloc, newloc_unref, wind);
+
+wind:
+        STACK_WIND (frame, default_rename_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+
+        if (oldloc_unref)
+                inode_unref (oldloc_unref);
+
+        if (newloc_unref)
+                inode_unref (newloc_unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+
+int
+ga_link (call_frame_t *frame, xlator_t *this,
+         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *oldloc_unref = NULL;
+        inode_t *newloc_unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
+        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, oldloc, oldloc_unref,
+                                         handle_newloc);
+
+handle_newloc:
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, newloc, newloc_unref, wind);
+
+wind:
+        STACK_WIND (frame, default_link_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+
+        if (oldloc_unref)
+                inode_unref (oldloc_unref);
+
+        if (newloc_unref)
+                inode_unref (newloc_unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            fd_t *fd, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        /* also check if the loc->inode itself is virtual
+           inode, if yes, return with failure, mainly because we
+           can't handle all the readdirp and other things on it. */
+        if (inode_ctx_get (loc->inode, this, NULL) == 0) {
+                op_errno = ENOTSUP;
+                goto err;
+        }
+
+        STACK_WIND (frame, default_opendir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             const char *name, dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+         dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid,
+            dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_gfid_access_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        " init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        ga_private_t *priv = NULL;
+        int ret = -1;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "not configured with exactly one child. exiting");
+                goto out;
+        }
+
+        /* This can be the top of graph in certain cases */
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dangling volume. check volfile ");
+        }
+
+        /* TODO: define a mem-type structure */
+        priv = GF_CALLOC (1, sizeof (*priv), gf_gfid_access_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        priv->newfile_args_pool = mem_pool_new (ga_newfile_args_t, 512);
+        if (!priv->newfile_args_pool)
+                goto out;
+
+        priv->heal_args_pool = mem_pool_new (ga_heal_args_t, 512);
+        if (!priv->heal_args_pool)
+                goto out;
+
+        this->local_pool = mem_pool_new (ga_local_t, 16);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
+        this->private = priv;
+
+        ret = 0;
+out:
+        if (ret && priv) {
+                if (priv->newfile_args_pool)
+                        mem_pool_destroy (priv->newfile_args_pool);
+                GF_FREE (priv);
+        }
+
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        ga_private_t *priv = NULL;
+        priv = this->private;
+        this->private = NULL;
+
+        if (priv) {
+                if (priv->newfile_args_pool)
+                        mem_pool_destroy (priv->newfile_args_pool);
+                if (priv->heal_args_pool)
+                        mem_pool_destroy (priv->heal_args_pool);
+                GF_FREE (priv);
+        }
+
+        return;
+}
+
+
+struct xlator_fops fops = {
+        .lookup = ga_lookup,
+
+        /* entry fops */
+        .mkdir   = ga_mkdir,
+        .mknod   = ga_mknod,
+        .create  = ga_create,
+        .symlink = ga_symlink,
+        .link    = ga_link,
+        .unlink  = ga_unlink,
+        .rmdir   = ga_rmdir,
+        .rename  = ga_rename,
+
+        /* handle any other directory operations here */
+        .opendir  = ga_opendir,
+        .stat     = ga_stat,
+        .setattr  = ga_setattr,
+        .getxattr = ga_getxattr,
+        .removexattr = ga_removexattr,
+
+        /* special fop to handle more entry creations */
+        .setxattr = ga_setxattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget = ga_forget,
+};
+
+struct volume_options options[] = {
+        /* This translator doesn't take any options, or provide any options */
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/gfid-access/src/gfid-access.h b/xlators/features/gfid-access/src/gfid-access.h
new file mode 100644
index 000000000..e883eca69
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access.h
@@ -0,0 +1,134 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __GFID_ACCESS_H__
+#define __GFID_ACCESS_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "gfid-access-mem-types.h"
+
+#define UUID_CANONICAL_FORM_LEN 36
+
+#define GF_FUSE_AUX_GFID_NEWFILE "glusterfs.gfid.newfile"
+#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+
+#define GF_GFID_KEY "GLUSTERFS_GFID"
+#define GF_GFID_DIR ".gfid"
+#define GF_AUX_GFID 0xd
+
+#define GFID_ACCESS_GET_VALID_DIR_INODE(x,l,unref,lbl) do {             \
+                int       ret       = 0;                                \
+                uint64_t  value     = 0;                                \
+                inode_t  *tmp_inode = NULL;                             \
+                                                                        \
+                /* if its an entry operation, on the virtual */         \
+                /* directory inode as parent, we need to handle */      \
+                /* it properly */                                       \
+                if (l->parent) {                                        \
+                        ret = inode_ctx_get (l->parent, x, &value);     \
+                        if (ret)                                        \
+                                goto lbl;                               \
+                        tmp_inode = (inode_t *)value;                   \
+                        l->parent = inode_ref (tmp_inode);                          \
+                        /* if parent is virtual, no need to handle */   \
+                        /* loc->inode */                                \
+                        break;                                          \
+                }                                                       \
+                                                                        \
+                /* if its an inode operation, on the virtual */         \
+                /* directory inode itself, we need to handle */         \
+                /* it properly */                                       \
+                if (l->inode) {                                         \
+                        ret = inode_ctx_get (l->inode, x, &value);      \
+                        if (ret)                                        \
+                                goto lbl;                               \
+                        tmp_inode = (inode_t *)value;                   \
+                        l->inode = inode_ref (tmp_inode);                           \
+                }                                                       \
+                                                                        \
+        } while (0)
+
+#define GFID_ACCESS_ENTRY_OP_CHECK(loc,err,lbl)    do {                 \
+                /* need to check if the lookup is on virtual dir */     \
+                if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&  \
+                    ((loc->parent &&                                    \
+                      __is_root_gfid (loc->parent->gfid)) ||            \
+                      __is_root_gfid (loc->pargfid))) {                 \
+                        err = EEXIST;                                   \
+                        goto lbl;                                       \
+                }                                                       \
+                                                                        \
+                /* now, check if the lookup() is on an existing */      \
+                /* entry, but on gfid-path */                           \
+                if ((loc->parent &&                                     \
+                     __is_gfid_access_dir (loc->parent->gfid)) ||       \
+                    __is_gfid_access_dir (loc->pargfid)) {              \
+                        err = EPERM;                                    \
+                        goto lbl;                                       \
+                }                                                       \
+        } while (0)
+
+
+typedef struct {
+        unsigned int  uid;
+        unsigned int  gid;
+        char          gfid[UUID_CANONICAL_FORM_LEN + 1];
+        unsigned int  st_mode;
+        char         *bname;
+
+        union {
+                struct _symlink_in {
+                        char     *linkpath;
+                } __attribute__ ((__packed__)) symlink;
+
+                struct _mknod_in {
+                        unsigned int mode;
+                        unsigned int rdev;
+                        unsigned int umask;
+                } __attribute__ ((__packed__)) mknod;
+
+                struct _mkdir_in {
+                        unsigned int mode;
+                        unsigned int umask;
+                } __attribute__ ((__packed__)) mkdir;
+        } __attribute__ ((__packed__)) args;
+} __attribute__((__packed__)) ga_newfile_args_t;
+
+typedef struct {
+        char      gfid[UUID_CANONICAL_FORM_LEN + 1];
+        char     *bname; /* a null terminated basename */
+} __attribute__((__packed__)) ga_heal_args_t;
+
+struct ga_private {
+        /* root inode's stbuf */
+        struct iatt root_stbuf;
+        struct iatt gfiddir_stbuf;
+        struct mem_pool *newfile_args_pool;
+        struct mem_pool *heal_args_pool;
+};
+typedef struct ga_private ga_private_t;
+
+struct __ga_local {
+        call_frame_t *orig_frame;
+        unsigned int uid;
+        unsigned int gid;
+        loc_t loc;
+};
+typedef struct __ga_local ga_local_t;
+
+#endif /* __GFID_ACCESS_H__ */
diff --git a/xlators/features/glupy/Makefile.am b/xlators/features/glupy/Makefile.am
new file mode 100644
index 000000000..060429ecf
--- /dev/null
+++ b/xlators/features/glupy/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src examples
+
+CLEANFILES =
diff --git a/xlators/features/glupy/doc/README.md b/xlators/features/glupy/doc/README.md
new file mode 100644
index 000000000..2d7b30ef6
--- /dev/null
+++ b/xlators/features/glupy/doc/README.md
@@ -0,0 +1,44 @@
+This is just the very start for a GlusterFS[1] meta-translator that will
+allow translator code to be written in Python.  It's based on the standard
+Python embedding (not extending) techniques, plus a dash of the ctypes module.
+The interface is a pretty minimal adaptation of the dispatches and callbacks
+from the C API[2] to Python, as follows:
+
+* Dispatch functions and callbacks must be defined on an "xlator" class
+  derived from gluster.Translator so that they'll be auto-registered with
+  the C translator during initialization.
+
+* For each dispatch or callback function you want to intercept, you define a
+  Python function using the xxx\_fop\_t or xxx\_cbk\_t decorator.
+
+* The arguments for each operation are different, so you'll need to refer to
+  the C API.  GlusterFS-specific types are used (though only loc\_t is fully
+  defined so far) and type correctness is enforced by ctypes.
+
+* If you do intercept a dispatch function, it is your responsibility to call
+  xxx\_wind (like STACK\_WIND in the C API but operation-specific) to pass
+  the request to the next translator.  If you do not intercept a function, it
+  will default the same way as for C (pass through to the same operation with
+  the same arguments on the first child translator).
+
+* If you intercept a callback function, it is your responsibility to call
+  xxx\_unwind (like STACK\_UNWIND\_STRICT in the C API) to pass the request back
+  to the caller.
+
+So far only the lookup and create operations are handled this way, to support
+the "negative lookup" example.  Now that the basic infrastructure is in place,
+adding more functions should be very quick, though with that much boilerplate I
+might pause to write a code generator.  I also plan to add structure
+definitions and interfaces for some of the utility functions in libglusterfs
+(especially those having to do with inode and fd context) in the fairly near
+future.  Note that you can also use ctypes to get at anything not explicitly
+exposed to Python already.
+
+_If you're coming here because of the Linux Journal article, please note that
+the code has evolved since that was written. The version that matches the
+article is here:_
+
+https://github.com/jdarcy/glupy/tree/4bbae91ba459ea46ef32f2966562492e4ca9187a
+
+[1] http://www.gluster.org
+[2] http://hekafs.org/dist/xlator_api_2.html
diff --git a/xlators/features/glupy/doc/TESTING b/xlators/features/glupy/doc/TESTING
new file mode 100644
index 000000000..e05f17f49
--- /dev/null
+++ b/xlators/features/glupy/doc/TESTING
@@ -0,0 +1,9 @@
+Loading a translator written in Python using the glupy meta translator
+-------------------------------------------------------------------------------
+'test.vol' is a simple volfile with the debug-trace Python translator on top
+of a brick. The volfile can be mounted using the following command.
+
+$ glusterfs --debug -f test.vol /path/to/mntpt
+
+If then file operations are performed on the newly mounted file system, log
+output would be printed by the Python translator on the standard output.
diff --git a/xlators/features/glupy/doc/test.vol b/xlators/features/glupy/doc/test.vol
new file mode 100644
index 000000000..0751a488c
--- /dev/null
+++ b/xlators/features/glupy/doc/test.vol
@@ -0,0 +1,10 @@
+volume vol-posix
+    type storage/posix
+    option directory /path/to/brick
+end-volume
+
+volume vol-glupy
+    type features/glupy
+    option module-name debug-trace
+    subvolumes vol-posix
+end-volume
diff --git a/xlators/features/glupy/examples/Makefile.am b/xlators/features/glupy/examples/Makefile.am
new file mode 100644
index 000000000..c26abeaaf
--- /dev/null
+++ b/xlators/features/glupy/examples/Makefile.am
@@ -0,0 +1,5 @@
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+glupyexamplesdir = $(xlatordir)/glupy
+
+glupyexamples_PYTHON = negative.py helloworld.py debug-trace.py
diff --git a/xlators/features/glupy/examples/debug-trace.py b/xlators/features/glupy/examples/debug-trace.py
new file mode 100644
index 000000000..6eef1b58b
--- /dev/null
+++ b/xlators/features/glupy/examples/debug-trace.py
@@ -0,0 +1,775 @@
+import sys
+import stat
+from uuid import UUID
+from time import strftime, localtime
+from gluster.glupy import *
+
+# This translator was written primarily to test the fop entry point definitions
+# and structure definitions in 'glupy.py'.
+
+# It is similar to the C language debug-trace translator, which logs the
+# arguments passed to the fops and their corresponding cbk functions.
+
+dl.get_id.restype = c_long
+dl.get_id.argtypes = [ POINTER(call_frame_t) ]
+
+dl.get_rootunique.restype = c_uint64
+dl.get_rootunique.argtypes = [ POINTER(call_frame_t) ]
+
+def uuid2str (gfid):
+        return str(UUID(''.join(map("{0:02x}".format, gfid))))
+
+
+def st_mode_from_ia (prot, filetype):
+        st_mode = 0
+        type_bit = 0
+        prot_bit = 0
+
+        if filetype == IA_IFREG:
+                type_bit = stat.S_IFREG
+        elif filetype == IA_IFDIR:
+                type_bit = stat.S_IFDIR
+        elif filetype == IA_IFLNK:
+                type_bit = stat.S_IFLNK
+        elif filetype == IA_IFBLK:
+                type_bit = stat.S_IFBLK
+        elif filetype == IA_IFCHR:
+                type_bit = stat.S_IFCHR
+        elif filetype == IA_IFIFO:
+                type_bit = stat.S_IFIFO
+        elif filetype == IA_IFSOCK:
+                type_bit = stat.S_IFSOCK
+        elif filetype == IA_INVAL:
+                pass
+
+
+        if prot.suid:
+                prot_bit |= stat.S_ISUID
+        if prot.sgid:
+                prot_bit |= stat.S_ISGID
+        if prot.sticky:
+                prot_bit |= stat.S_ISVTX
+
+        if prot.owner.read:
+                prot_bit |= stat.S_IRUSR
+        if prot.owner.write:
+                prot_bit |= stat.S_IWUSR
+        if prot.owner.execn:
+                prot_bit |= stat.S_IXUSR
+
+        if prot.group.read:
+                prot_bit |= stat.S_IRGRP
+        if prot.group.write:
+                prot_bit |= stat.S_IWGRP
+        if prot.group.execn:
+                prot_bit |= stat.S_IXGRP
+
+        if prot.other.read:
+                prot_bit |= stat.S_IROTH
+        if prot.other.write:
+                prot_bit |= stat.S_IWOTH
+        if prot.other.execn:
+                prot_bit |= stat.S_IXOTH
+
+        st_mode = (type_bit | prot_bit)
+
+        return st_mode
+
+
+def trace_stat2str (buf):
+        gfid = uuid2str(buf.contents.ia_gfid)
+        mode = st_mode_from_ia(buf.contents.ia_prot, buf.contents.ia_type)
+        atime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_atime))
+        mtime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_mtime))
+        ctime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_ctime))
+        return ("(gfid={0:s}, ino={1:d}, mode={2:o}, nlink={3:d}, uid ={4:d}, "+
+                "gid ={5:d}, size={6:d}, blocks={7:d}, atime={8:s}, mtime={9:s}, "+
+                "ctime={10:s})").format(gfid, buf.contents.ia_no, mode,
+                                        buf.contents.ia_nlink,
+                                        buf.contents.ia_uid,
+                                        buf.contents.ia_gid,
+                                        buf.contents.ia_size,
+                                        buf.contents.ia_blocks,
+                                        atime_buf, mtime_buf,
+                                        ctime_buf)
+
+class xlator(Translator):
+
+        def __init__(self, c_this):
+                Translator.__init__(self, c_this)
+                self.gfids = {}
+
+        def lookup_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.gfid)
+                print("GLUPY TRACE LOOKUP FOP- {0:d}: gfid={1:s}; " +
+                      "path={2:s}").format(unique, gfid, loc.contents.path)
+                self.gfids[key] = gfid
+                dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def lookup_cbk(self, frame, cookie, this, op_ret, op_errno,
+                       inode, buf, xdata, postparent):
+                unique =dl.get_rootunique(frame)
+                key =dl.get_id(frame)
+                if op_ret == 0:
+                        gfid = uuid2str(buf.contents.ia_gfid)
+                        statstr = trace_stat2str(buf)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE LOOKUP CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *buf={3:s}; " +
+                              "*postparent={4:s}").format(unique, gfid,
+                                                          op_ret, statstr,
+                                                          postparentstr)
+                else:
+                        gfid = self.gfids[key]
+                        print("GLUPY TRACE LOOKUP CBK - {0:d}: gfid={1:s};" +
+                              " op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                      gfid,
+                                                                      op_ret,
+                                                                      op_errno)
+                del self.gfids[key]
+                dl.unwind_lookup(frame, cookie, this, op_ret, op_errno,
+                                 inode, buf, xdata, postparent)
+                return 0
+
+        def create_fop(self, frame, this, loc, flags, mode, umask, fd,
+                       xdata):
+                unique = dl.get_rootunique(frame)
+                gfid = uuid2str(loc.contents.gfid)
+                print("GLUPY TRACE CREATE FOP- {0:d}: gfid={1:s}; path={2:s}; " +
+                      "fd={3:s}; flags=0{4:o}; mode=0{5:o}; " +
+                      "umask=0{6:o}").format(unique, gfid, loc.contents.path,
+                                             fd, flags, mode, umask)
+                dl.wind_create(frame, POINTER(xlator_t)(), loc, flags,mode,
+                               umask, fd, xdata)
+                return 0
+
+        def create_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
+                       inode, buf, preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret >= 0:
+                        gfid = uuid2str(inode.contents.gfid)
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE CREATE CBK- {0:d}: gfid={1:s};" +
+                              " op_ret={2:d}; fd={3:s}; *stbuf={4:s}; " +
+                              "*preparent={5:s};" +
+                              " *postparent={6:s}").format(unique, gfid, op_ret,
+                                                           fd, statstr,
+                                                           preparentstr,
+                                                           postparentstr)
+                else:
+                        print ("GLUPY TRACE CREATE CBK- {0:d}: op_ret={1:d}; " +
+                              "op_errno={2:d}").format(unique, op_ret, op_errno)
+                dl.unwind_create(frame, cookie, this, op_ret, op_errno, fd,
+                                 inode, buf, preparent, postparent, xdata)
+                return 0
+
+        def open_fop(self, frame, this, loc, flags, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE OPEN FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "flags={3:d}; fd={4:s}").format(unique, gfid,
+                                                      loc.contents.path, flags,
+                                                      fd)
+                self.gfids[key] = gfid
+                dl.wind_open(frame, POINTER(xlator_t)(), loc, flags, fd, xdata)
+                return 0
+
+        def open_cbk(self, frame, cookie, this, op_ret, op_errno, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE OPEN CBK- {0:d}: gfid={1:s}; op_ret={2:d}; "
+                      "op_errno={3:d}; *fd={4:s}").format(unique, gfid,
+                                                          op_ret, op_errno, fd)
+                del self.gfids[key]
+                dl.unwind_open(frame, cookie, this, op_ret, op_errno, fd,
+                               xdata)
+                return 0
+
+        def readv_fop(self, frame, this, fd, size, offset, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READV FOP- {0:d}: gfid={1:s}; "+
+                      "fd={2:s}; size ={3:d}; offset={4:d}; " +
+                      "flags=0{5:x}").format(unique, gfid, fd, size, offset,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_readv (frame, POINTER(xlator_t)(), fd, size, offset,
+                               flags, xdata)
+                return 0
+
+        def readv_cbk(self, frame, cookie, this, op_ret, op_errno, vector,
+                      count, buf, iobref, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret >= 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
+                              "op_ret={2:d}; *buf={3:s};").format(unique, gfid,
+                                                                  op_ret,
+                                                                  statstr)
+
+                else:
+                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_readv (frame, cookie, this, op_ret, op_errno,
+                                 vector, count, buf, iobref, xdata)
+                return 0
+
+        def writev_fop(self, frame, this, fd, vector, count, offset, flags,
+                       iobref, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE  WRITEV FOP- {0:d}: gfid={1:s}; " +
+                      "fd={2:s}; count={3:d}; offset={4:d}; " +
+                      "flags=0{5:x}").format(unique, gfid, fd, count, offset,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_writev(frame, POINTER(xlator_t)(), fd, vector, count,
+                               offset, flags, iobref, xdata)
+                return 0
+
+        def writev_cbk(self, frame, cookie, this, op_ret, op_errno, prebuf,
+                       postbuf, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                if op_ret >= 0:
+                        preopstr = trace_stat2str(prebuf)
+                        postopstr = trace_stat2str(postbuf)
+                        print("GLUPY TRACE WRITEV CBK- {0:d}: op_ret={1:d}; " +
+                              "*prebuf={2:s}; " +
+                              "*postbuf={3:s}").format(unique, op_ret, preopstr,
+                                                       postopstr)
+                else:
+                        gfid = self.gfids[key]
+                        print("GLUPY TRACE WRITEV CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_writev (frame, cookie, this, op_ret, op_errno,
+                                  prebuf, postbuf, xdata)
+                return 0
+
+        def opendir_fop(self, frame, this, loc, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE OPENDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "fd={3:s}").format(unique, gfid, loc.contents.path, fd)
+                self.gfids[key] = gfid
+                dl.wind_opendir(frame, POINTER(xlator_t)(), loc, fd, xdata)
+                return 0
+
+        def opendir_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
+                        xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE OPENDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
+                      " op_errno={3:d}; fd={4:s}").format(unique, gfid, op_ret,
+                                                          op_errno, fd)
+                del self.gfids[key]
+                dl.unwind_opendir(frame, cookie, this, op_ret, op_errno,
+                                  fd, xdata)
+                return 0
+
+        def readdir_fop(self, frame, this, fd, size, offset, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READDIR FOP- {0:d}:  gfid={1:s}; fd={2:s}; " +
+                      "size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
+                                                         offset)
+                self.gfids[key] = gfid
+                dl.wind_readdir(frame, POINTER(xlator_t)(), fd, size, offset,
+                                xdata)
+                return 0
+
+        def readdir_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                        xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE READDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
+                      " op_errno={3:d}").format(unique, gfid, op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_readdir(frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata)
+                return 0
+
+        def readdirp_fop(self, frame, this, fd, size, offset, dictionary):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READDIRP FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
+                      " size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
+                                                          offset)
+                self.gfids[key] = gfid
+                dl.wind_readdirp(frame, POINTER(xlator_t)(), fd, size, offset,
+                                 dictionary)
+                return 0
+
+        def readdirp_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                         xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE READDIRP CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_readdirp(frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata)
+                return 0
+
+        def mkdir_fop(self, frame, this, loc, mode, umask, xdata):
+                unique = dl.get_rootunique(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE MKDIR FOP- {0:d}: gfid={1:s}; path={2:s}; " +
+                      "mode={3:d}; umask=0{4:o}").format(unique, gfid,
+                                                         loc.contents.path, mode,
+                                                         umask)
+                dl.wind_mkdir(frame, POINTER(xlator_t)(), loc, mode, umask,
+                              xdata)
+                return 0
+
+        def mkdir_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                      preparent, postparent,  xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret == 0:
+                        gfid = uuid2str(inode.contents.gfid)
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE MKDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *stbuf={3:s}; *prebuf={4:s}; "+
+                              "*postbuf={5:s} ").format(unique, gfid, op_ret,
+                                                        statstr,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE MKDIR CBK- {0:d}:  op_ret={1:d}; "+
+                              "op_errno={2:d}").format(unique, op_ret, op_errno)
+                dl.unwind_mkdir(frame, cookie, this, op_ret, op_errno, inode,
+                                buf, preparent, postparent, xdata)
+                return 0
+
+        def rmdir_fop(self, frame, this, loc, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE RMDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "flags={3:d}").format(unique, gfid, loc.contents.path,
+                                            flags)
+                self.gfids[key] = gfid
+                dl.wind_rmdir(frame, POINTER(xlator_t)(), loc, flags, xdata)
+                return 0
+
+        def rmdir_cbk(self, frame, cookie, this, op_ret, op_errno, preparent,
+                      postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *prebuf={3:s}; "+
+                              "*postbuf={4:s}").format(unique, gfid, op_ret,
+                                                       preparentstr,
+                                                       postparentstr)
+                else:
+                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_rmdir(frame, cookie, this, op_ret, op_errno,
+                                preparent, postparent, xdata)
+                return 0
+
+        def stat_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE STAT FOP- {0:d}: gfid={1:s}; " +
+                      " path={2:s}").format(unique, gfid, loc.contents.path)
+                self.gfids[key] = gfid
+                dl.wind_stat(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def stat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                     xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d};  *buf={3:s};").format(unique,
+                                                                   gfid,
+                                                                   op_ret,
+                                                                   statstr)
+                else:
+                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_stat(frame, cookie, this, op_ret, op_errno,
+                               buf, xdata)
+                return 0
+
+        def fstat_fop(self, frame, this, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FSTAT FOP- {0:d}:  gfid={1:s}; " +
+                      "fd={2:s}").format(unique, gfid, fd)
+                self.gfids[key] = gfid
+                dl.wind_fstat(frame, POINTER(xlator_t)(), fd, xdata)
+                return 0
+
+        def fstat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                      xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; *buf={3:s}").format(unique,
+                                                                  gfid,
+                                                                  op_ret,
+                                                                  statstr)
+                else:
+                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique.
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_fstat(frame, cookie, this, op_ret, op_errno,
+                                buf, xdata)
+                return 0
+
+        def statfs_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                if loc.contents.inode:
+                        gfid = uuid2str(loc.contents.inode.contents.gfid)
+                else:
+                        gfid = "0"
+                print("GLUPY TRACE STATFS FOP- {0:d}: gfid={1:s}; "+
+                      "path={2:s}").format(unique, gfid, loc.contents.path)
+                dl.wind_statfs(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def statfs_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                       xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret == 0:
+                        #TBD: print buf (pointer to an iovec type object)
+                        print("GLUPY TRACE STATFS CBK {0:d}: "+
+                              "op_ret={1:d}").format(unique, op_ret)
+                else:
+                        print("GLUPY TRACE STATFS CBK-  {0:d}"+
+                              "op_ret={1:d}; op_errno={2:d}").format(unique,
+                                                                     op_ret,
+                                                                     op_errno)
+                dl.unwind_statfs(frame, cookie, this, op_ret, op_errno,
+                                 buf, xdata)
+                return 0
+
+        def getxattr_fop(self, frame, this, loc, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE GETXATTR FOP- {0:d}: gfid={1:s}; path={2:s};"+
+                      " name={3:s}").format(unique, gfid, loc.contents.path,
+                                            name)
+                self.gfids[key]=gfid
+                dl.wind_getxattr(frame, POINTER(xlator_t)(), loc, name, xdata)
+                return 0
+
+        def getxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                         dictionary, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE GETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}; "+
+                      " dictionary={4:s}").format(unique, gfid, op_ret, op_errno,
+                                                  dictionary)
+                del self.gfids[key]
+                dl.unwind_getxattr(frame, cookie, this, op_ret, op_errno,
+                                   dictionary, xdata)
+                return 0
+
+        def fgetxattr_fop(self, frame, this, fd, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FGETXATTR FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
+                      "name={3:s}").format(unique, gfid, fd, name)
+                self.gfids[key] = gfid
+                dl.wind_fgetxattr(frame, POINTER(xlator_t)(), fd, name, xdata)
+                return 0
+
+        def fgetxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                          dictionary, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE FGETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d};"+
+                      " dictionary={4:s}").format(unique, gfid, op_ret,
+                                                  op_errno, dictionary)
+                del self.gfids[key]
+                dl.unwind_fgetxattr(frame, cookie, this, op_ret, op_errno,
+                                    dictionary, xdata)
+                return 0
+
+        def setxattr_fop(self, frame, this, loc, dictionary, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE SETXATTR FOP- {0:d}:  gfid={1:s}; path={2:s};"+
+                      " flags={3:d}").format(unique, gfid, loc.contents.path,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_setxattr(frame, POINTER(xlator_t)(), loc, dictionary,
+                                 flags, xdata)
+                return 0
+
+        def setxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE SETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_setxattr(frame, cookie, this, op_ret, op_errno,
+                                   xdata)
+                return 0
+
+        def fsetxattr_fop(self, frame, this, fd, dictionary, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FSETXATTR FOP- {0:d}: gfid={1:s}; fd={2:p}; "+
+                      "flags={3:d}").format(unique, gfid, fd, flags)
+                self.gfids[key] = gfid
+                dl.wind_fsetxattr(frame, POINTER(xlator_t)(), fd, dictionary,
+                                  flags, xdata)
+                return 0
+
+        def fsetxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE FSETXATTR CBK- {0:d}: gfid={1:s};  "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_fsetxattr(frame, cookie, this, op_ret, op_errno,
+                                   xdata)
+                return 0
+
+        def removexattr_fop(self, frame, this, loc, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE REMOVEXATTR FOP- {0:d}:  gfid={1:s}; "+
+                      "path={2:s}; name={3:s}").format(unique, gfid,
+                                                       loc.contents.path,
+                                                       name)
+                self.gfids[key] = gfid
+                dl.wind_removexattr(frame, POINTER(xlator_t)(), loc, name,
+                                    xdata)
+                return 0
+
+        def removexattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                            xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE REMOVEXATTR CBK- {0:d}: gfid={1:s} "+
+                      " op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                              op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_removexattr(frame, cookie, this, op_ret, op_errno,
+                                      xdata)
+                return 0
+
+        def link_fop(self, frame, this, oldloc, newloc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                if (newloc.contents.inode):
+                        newgfid = uuid2str(newloc.contents.inode.contents.gfid)
+                else:
+                        newgfid = "0"
+                oldgfid = uuid2str(oldloc.contents.inode.contents.gfid)
+                print("GLUPY TRACE LINK FOP-{0:d}: oldgfid={1:s}; oldpath={2:s};"+
+                      "newgfid={3:s};"+
+                      "newpath={4:s}").format(unique, oldgfid,
+                                              oldloc.contents.path,
+                                              newgfid,
+                                              newloc.contents.path)
+                self.gfids[key] =  oldgfid
+                dl.wind_link(frame, POINTER(xlator_t)(), oldloc, newloc,
+                             xdata)
+                return 0
+
+        def link_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                     preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE LINK CBK- {0:d}: op_ret={1:d} "+
+                              "*stbuf={2:s}; *prebuf={3:s}; "+
+                              "*postbuf={4:s} ").format(unique, op_ret, statstr,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE LINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; "+
+                              "op_errno={3:d}").format(unique, gfid,
+                                                       op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_link(frame, cookie, this, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata)
+                return 0
+
+        def unlink_fop(self, frame, this, loc, xflag, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE UNLINK FOP- {0:d}; gfid={1:s}; path={2:s}; "+
+                      "flag={3:d}").format(unique, gfid, loc.contents.path,
+                                           xflag)
+                self.gfids[key] = gfid
+                dl.wind_unlink(frame, POINTER(xlator_t)(), loc, xflag,
+                               xdata)
+                return 0
+
+        def unlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                       preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE UNLINK CBK- {0:d}: gfid ={1:s}; "+
+                              "op_ret={2:d}; *prebuf={3:s}; "+
+                              "*postbuf={4:s} ").format(unique, gfid, op_ret,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE UNLINK CBK: {0:d}: gfid ={1:s}; "+
+                              "op_ret={2:d}; "+
+                              "op_errno={3:d}").format(unique, gfid, op_ret,
+                                                       op_errno)
+                del self.gfids[key]
+                dl.unwind_unlink(frame, cookie, this, op_ret, op_errno,
+                                 preparent, postparent, xdata)
+                return 0
+
+        def readlink_fop(self, frame, this, loc, size, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE READLINK FOP- {0:d}:  gfid={1:s}; path={2:s};"+
+                      " size={3:d}").format(unique, gfid, loc.contents.path,
+                                            size)
+                self.gfids[key] = gfid
+                dl.wind_readlink(frame, POINTER(xlator_t)(), loc, size,
+                               xdata)
+                return 0
+
+        def readlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                         buf, stbuf, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid  = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(stbuf)
+                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; op_errno={3:d}; *prebuf={4:s}; "+
+                              "*postbuf={5:s} ").format(unique, gfid,
+                                                        op_ret, op_errno,
+                                                        buf, statstr)
+                else:
+                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                      gfid,
+                                                                      op_ret,
+                                                                      op_errno)
+                del self.gfids[key]
+                dl.unwind_readlink(frame, cookie, this, op_ret, op_errno, buf,
+                                   stbuf, xdata)
+                return 0
+
+        def symlink_fop(self, frame, this, linkpath, loc, umask, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE SYMLINK FOP- {0:d}: gfid={1:s}; "+
+                      "linkpath={2:s}; path={3:s};"+
+                      "umask=0{4:o}").format(unique, gfid, linkpath,
+                                             loc.contents.path, umask)
+                self.gfids[key] = gfid
+                dl.wind_symlink(frame, POINTER(xlator_t)(), linkpath, loc,
+                                umask, xdata)
+                return 0
+
+        def symlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                        inode, buf, preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *stbuf={3:s}; *preparent={4:s}; "+
+                              "*postparent={5:s}").format(unique, gfid,
+                                                          op_ret, statstr,
+                                                          preparentstr,
+                                                          postparentstr)
+                else:
+                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_symlink(frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata)
+                return 0
diff --git a/xlators/features/glupy/examples/helloworld.py b/xlators/features/glupy/examples/helloworld.py
new file mode 100644
index 000000000..b565a4e5b
--- /dev/null
+++ b/xlators/features/glupy/examples/helloworld.py
@@ -0,0 +1,19 @@
+import sys
+from gluster.glupy import *
+
+class xlator (Translator):
+
+    def __init__(self, c_this):
+        Translator.__init__(self, c_this)
+
+    def lookup_fop(self, frame, this, loc, xdata):
+        print "Python xlator: Hello!"
+        dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
+        return 0
+
+    def lookup_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                   xdata, postparent):
+        print "Python xlator: Hello again!"
+        dl.unwind_lookup(frame, cookie, this, op_ret, op_errno, inode, buf,
+                         xdata, postparent)
+        return 0
diff --git a/xlators/features/glupy/examples/negative.py b/xlators/features/glupy/examples/negative.py
new file mode 100644
index 000000000..e7a4fc07c
--- /dev/null
+++ b/xlators/features/glupy/examples/negative.py
@@ -0,0 +1,91 @@
+import sys
+from uuid import UUID
+from gluster.glupy import *
+
+# Negative-lookup-caching example.  If a file wasn't there the last time we
+# looked, it's probably still not there.  This translator keeps track of
+# those failed lookups for us, and returns ENOENT without needing to pass the
+# call any further for repeated requests.
+
+# If we were doing this for real, we'd need separate caches for each xlator
+# instance.  The easiest way to do this would be to have xlator.__init__
+# "register" each instance in a module-global dict, with the key as the C
+# translator address and the value as the xlator object itself.  For testing
+# and teaching, it's sufficient just to have one cache.  The keys are parent
+# GFIDs, and the entries are lists of names within that parent that we know
+# don't exist.
+cache = {}
+
+# TBD: we need a better way of handling per-request data (frame->local in C).
+dl.get_id.restype = c_long
+dl.get_id.argtypes = [ POINTER(call_frame_t) ]
+
+def uuid2str (gfid):
+    return str(UUID(''.join(map("{0:02x}".format, gfid))))
+
+class xlator (Translator):
+
+    def __init__ (self, c_this):
+        self.requests = {}
+        Translator.__init__(self,c_this)
+
+    def lookup_fop (self, frame, this, loc, xdata):
+        pargfid = uuid2str(loc.contents.pargfid)
+        print "lookup FOP: %s:%s" % (pargfid, loc.contents.name)
+        # Check the cache.
+        if cache.has_key(pargfid):
+            if loc.contents.name in cache[pargfid]:
+                print "short-circuiting for %s:%s" % (pargfid,
+                    loc.contents.name)
+                dl.unwind_lookup(frame,0,this,-1,2,None,None,None,None)
+                return 0
+        key = dl.get_id(frame)
+        self.requests[key] = (pargfid, loc.contents.name[:])
+        # TBD: get real child xl from init, pass it here
+        dl.wind_lookup(frame,POINTER(xlator_t)(),loc,xdata)
+        return 0
+
+    def lookup_cbk (self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                    xdata, postparent):
+        print "lookup CBK: %d (%d)" % (op_ret, op_errno)
+        key = dl.get_id(frame)
+        pargfid, name = self.requests[key]
+        # Update the cache.
+        if op_ret == 0:
+            print "found %s, removing from cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].discard(name)
+        elif op_errno == 2:    # ENOENT
+            print "failed to find %s, adding to cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].add(name)
+            else:
+                cache[pargfid] = set([name])
+        del self.requests[key]
+        dl.unwind_lookup(frame,cookie,this,op_ret,op_errno,
+                         inode,buf,xdata,postparent)
+        return 0
+
+    def create_fop (self, frame, this, loc, flags, mode, umask, fd, xdata):
+        pargfid = uuid2str(loc.contents.pargfid)
+        print "create FOP: %s:%s" % (pargfid, loc.contents.name)
+        key = dl.get_id(frame)
+        self.requests[key] = (pargfid, loc.contents.name[:])
+        # TBD: get real child xl from init, pass it here
+        dl.wind_create(frame,POINTER(xlator_t)(),loc,flags,mode,umask,fd,xdata)
+        return 0
+
+    def create_cbk (self, frame, cookie, this, op_ret, op_errno, fd, inode,
+                    buf, preparent, postparent, xdata):
+        print "create CBK: %d (%d)" % (op_ret, op_errno)
+        key = dl.get_id(frame)
+        pargfid, name = self.requests[key]
+        # Update the cache.
+        if op_ret == 0:
+            print "created %s, removing from cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].discard(name)
+        del self.requests[key]
+        dl.unwind_create(frame,cookie,this,op_ret,op_errno,fd,inode,buf,
+                         preparent,postparent,xdata)
+        return 0
diff --git a/xlators/features/glupy/src/Makefile.am b/xlators/features/glupy/src/Makefile.am
new file mode 100644
index 000000000..ae7b6d14d
--- /dev/null
+++ b/xlators/features/glupy/src/Makefile.am
@@ -0,0 +1,21 @@
+xlator_LTLIBRARIES = glupy.la
+
+# Ensure GLUSTER_PYTHON_PATH is passed to glupy.so
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+glupydir = $(xlatordir)/glupy
+AM_CPPFLAGS = $(PYTHONDEV_CPPFLAGS) $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -isystem $(BUILD_PYTHON_INC)
+AM_CFLAGS = $(PYTHONDEV_CFLAGS) -Wall -fno-strict-aliasing -DGLUSTER_PYTHON_PATH=\"$(glupydir)\" $(GF_CFLAGS)
+
+# Flags to build glupy.so with
+glupy_la_LDFLAGS = $(PYTHONDEV_LDFLAGS) -module -avoid-version -shared -nostartfiles
+glupy_la_SOURCES = glupy.c
+glupy_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	 -lpthread -l$(BUILD_PYTHON_LIB)
+
+noinst_HEADERS = glupy.h
+
+# Install glupy.py into the Python site-packages area
+pyglupydir = $(pythondir)/gluster
+pyglupy_PYTHON = glupy.py
+
+CLEANFILES =
diff --git a/xlators/features/glupy/src/glupy.c b/xlators/features/glupy/src/glupy.c
new file mode 100644
index 000000000..948b66f8d
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.c
@@ -0,0 +1,2471 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  GlusterFS is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published
+  by the Free Software Foundation; either version 3 of the License,
+  or (at your option) any later version.
+
+  GlusterFS is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see
+  <http://www.gnu.org/licenses/>.
+*/
+
+#include <ctype.h>
+#include <sys/uio.h>
+#include <Python.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "glupy.h"
+
+/* UTILITY FUNCTIONS FOR FOP-SPECIFIC CODE */
+
+pthread_key_t gil_init_key;
+
+PyGILState_STATE
+glupy_enter (void)
+{
+#if 0
+        if (!pthread_getspecific(gil_init_key)) {
+                PyEval_ReleaseLock();
+                (void)pthread_setspecific(gil_init_key,(void *)1);
+        }
+#endif
+
+        return PyGILState_Ensure();
+}
+
+void
+glupy_leave (PyGILState_STATE gstate)
+{
+        PyGILState_Release(gstate);
+}
+
+/* FOP: LOOKUP */
+
+int32_t
+glupy_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_LOOKUP]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_lookup_cbk_t)(priv->cbks[GLUPY_LOOKUP]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, xdata, postparent);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
+        return 0;
+}
+
+int32_t
+glupy_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_LOOKUP]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_lookup_t)(priv->fops[GLUPY_LOOKUP]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+}
+
+void
+wind_lookup (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_lookup_cbk,xl,xl->fops->lookup,loc,xdata);
+}
+
+void
+unwind_lookup (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(lookup,frame,op_ret,op_errno,
+                            inode,buf,xdata,postparent);
+}
+
+void
+set_lookup_fop (long py_this, fop_lookup_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_LOOKUP] = (long)fop;
+}
+
+void
+set_lookup_cbk (long py_this, fop_lookup_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_LOOKUP] = (long)cbk;
+}
+
+/* FOP: CREATE */
+
+int32_t
+glupy_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_CREATE]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_create_cbk_t)(priv->cbks[GLUPY_CREATE]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_CREATE]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_create_t)(priv->fops[GLUPY_CREATE]))(
+                frame, this, loc, flags, mode, umask, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_create_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
+        return 0;
+}
+
+void
+wind_create (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_create_cbk,xl, xl->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+}
+
+void
+unwind_create (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_create_fop (long py_this, fop_create_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_CREATE] = (long)fop;
+}
+
+void
+set_create_cbk (long py_this, fop_create_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_CREATE] = (long)cbk;
+}
+
+/* FOP: OPEN */
+
+int32_t
+glupy_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_OPEN]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_open_cbk_t)(priv->cbks[GLUPY_OPEN]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+glupy_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            int32_t flags, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_OPEN]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_open_t)(priv->fops[GLUPY_OPEN]))(
+                frame, this, loc, flags, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_open_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+        return 0;
+}
+
+void
+wind_open (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_open_cbk, xl, xl->fops->open, loc, flags,
+                    fd, xdata);
+}
+
+void
+unwind_open (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+}
+
+void
+set_open_fop (long py_this, fop_open_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_OPEN] = (long)fop;
+}
+
+void
+set_open_cbk (long py_this, fop_open_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_OPEN] = (long)cbk;
+}
+
+/* FOP: READV */
+
+int32_t
+glupy_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                 int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                 dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READV]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readv_cbk_t)(priv->cbks[GLUPY_READV]))(
+                frame, cookie, this, op_ret, op_errno,
+                vector, count, stbuf, iobref, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+glupy_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READV]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readv_t)(priv->fops[GLUPY_READV]))(
+                frame, this, fd, size, offset, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_readv (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_readv_cbk, xl, xl->fops->readv, fd, size,
+                    offset, flags, xdata);
+}
+
+void
+unwind_readv (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iovec *vector,
+              int32_t count, struct iatt *stbuf, struct iobref *iobref,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+}
+
+void
+set_readv_fop (long py_this, fop_readv_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_READV] = (long)fop;
+}
+
+void
+set_readv_cbk (long py_this, fop_readv_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_READV] = (long)cbk;
+}
+
+/* FOP: WRITEV */
+
+int32_t
+glupy_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_WRITEV]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_writev_cbk_t)(priv->cbks[GLUPY_WRITEV]))(
+                frame, cookie, this, op_ret, op_errno,
+                prebuf, postbuf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count, off_t offset,
+              uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_WRITEV]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_writev_t)(priv->fops[GLUPY_WRITEV]))(
+                frame, this, fd, vector, count, offset, flags,
+                iobref, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_writev_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                    offset, flags, iobref, xdata);
+        return 0;
+}
+
+void
+wind_writev (call_frame_t *frame, xlator_t *xl, fd_t *fd, struct iovec *vector,
+             int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+             dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_writev_cbk, xl, xl->fops->writev, fd, vector,
+                    count, offset, flags, iobref, xdata);
+}
+
+void
+unwind_writev (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+}
+
+void
+set_writev_fop (long py_this, fop_writev_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_WRITEV] = (long)fop;
+}
+
+void
+set_writev_cbk (long py_this, fop_writev_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_WRITEV] = (long)cbk;
+}
+
+
+/* FOP: OPENDIR */
+
+int32_t
+glupy_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd,
+                   dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_OPENDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_opendir_cbk_t)(priv->cbks[GLUPY_OPENDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+glupy_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_OPENDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_opendir_t)(priv->fops[GLUPY_OPENDIR]))(
+                frame, this, loc, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_opendir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+        return 0;
+}
+
+void
+wind_opendir (call_frame_t *frame, xlator_t *xl, loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_opendir_cbk,xl,xl->fops->opendir,loc,fd,xdata);
+}
+
+void
+unwind_opendir (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(opendir,frame,op_ret,op_errno,
+                            fd,xdata);
+}
+
+void
+set_opendir_fop (long py_this, fop_opendir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_OPENDIR] = (long)fop;
+}
+
+void
+set_opendir_cbk (long py_this, fop_opendir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_OPENDIR] = (long)cbk;
+}
+
+/* FOP: READDIR */
+
+int32_t
+glupy_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                   dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readdir_cbk_t)(priv->cbks[GLUPY_READDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                entries,  xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               size_t size, off_t offset, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readdir_t)(priv->fops[GLUPY_READDIR]))(
+                frame, this, fd, size, offset, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir,fd, size, offset, xdata);
+        return 0;
+}
+
+void
+wind_readdir(call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_readdir_cbk,xl,xl->fops->readdir,fd,size,offset,xdata);
+}
+
+void
+unwind_readdir (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(readdir,frame,op_ret,op_errno,
+                            entries, xdata);
+}
+
+void
+set_readdir_fop (long py_this, fop_readdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READDIR] = (long)fop;
+}
+
+void
+set_readdir_cbk (long py_this, fop_readdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READDIR] = (long)cbk;
+}
+
+
+/* FOP: READDIRP */
+
+int32_t
+glupy_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READDIRP]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readdirp_cbk_t)(priv->cbks[GLUPY_READDIRP]))(
+                frame, cookie, this, op_ret, op_errno,
+                entries,  xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                size_t size, off_t offset, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READDIRP]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readdirp_t)(priv->fops[GLUPY_READDIRP]))(
+                frame, this, fd, size, offset, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readdirp_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp,fd, size, offset, xdata);
+        return 0;
+}
+
+void
+wind_readdirp (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_readdirp_cbk,xl,xl->fops->readdirp,fd,size,offset,xdata);
+}
+
+void
+unwind_readdirp (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                 dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(readdirp,frame,op_ret,op_errno,
+                            entries, xdata);
+}
+
+void
+set_readdirp_fop (long py_this, fop_readdirp_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READDIRP] = (long)fop;
+}
+
+void
+set_readdirp_cbk (long py_this, fop_readdirp_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READDIRP] = (long)cbk;
+}
+
+
+/* FOP:STAT */
+
+int32_t
+glupy_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_STAT]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_stat_cbk_t)(priv->cbks[GLUPY_STAT]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_STAT]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_stat_t)(priv->fops[GLUPY_STAT]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+}
+
+void
+wind_stat (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_stat_cbk,xl,xl->fops->stat,loc,xdata);
+}
+
+void
+unwind_stat (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, struct iatt *buf,
+             dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(stat,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_stat_fop (long py_this, fop_stat_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_STAT] = (long)fop;
+}
+
+void
+set_stat_cbk (long py_this, fop_stat_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_STAT] = (long)cbk;
+}
+
+
+/* FOP: FSTAT */
+
+int32_t
+glupy_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FSTAT]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fstat_cbk_t)(priv->cbks[GLUPY_FSTAT]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FSTAT]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fstat_t)(priv->fops[GLUPY_FSTAT]))(
+                frame, this, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+}
+
+void
+wind_fstat (call_frame_t *frame, xlator_t *xl, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_fstat_cbk,xl,xl->fops->fstat,fd,xdata);
+}
+
+void
+unwind_fstat (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iatt *buf,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(fstat,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_fstat_fop (long py_this, fop_fstat_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FSTAT] = (long)fop;
+}
+
+void
+set_fstat_cbk (long py_this, fop_fstat_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FSTAT] = (long)cbk;
+}
+
+/* FOP:STATFS */
+
+int32_t
+glupy_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_STATFS]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_statfs_cbk_t)(priv->cbks[GLUPY_STATFS]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_STATFS]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_statfs_t)(priv->fops[GLUPY_STATFS]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_statfs_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+}
+
+void
+wind_statfs (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_statfs_cbk,xl,xl->fops->statfs,loc,xdata);
+}
+
+void
+unwind_statfs (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+               dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(statfs,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_statfs_fop (long py_this, fop_statfs_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_STATFS] = (long)fop;
+}
+
+void
+set_statfs_cbk (long py_this, fop_statfs_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_STATFS] = (long)cbk;
+}
+
+
+/* FOP: SETXATTR */
+
+int32_t
+glupy_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_SETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_setxattr_cbk_t)(priv->cbks[GLUPY_SETXATTR]))(
+                frame, cookie, this, op_ret, op_errno,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_SETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_setxattr_t)(priv->fops[GLUPY_SETXATTR]))(
+                frame, this, loc, dict, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_setxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_setxattr_cbk, xl, xl->fops->setxattr,
+                    loc, dict, flags, xdata);
+}
+
+
+void
+unwind_setxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_setxattr_fop (long py_this, fop_setxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_SETXATTR] = (long)fop;
+}
+
+void
+set_setxattr_cbk (long py_this, fop_setxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_SETXATTR] = (long)cbk;
+}
+
+/* FOP: GETXATTR */
+
+int32_t
+glupy_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_GETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_getxattr_cbk_t)(priv->cbks[GLUPY_GETXATTR]))(
+                frame, cookie, this, op_ret, op_errno, dict,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_GETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_getxattr_t)(priv->fops[GLUPY_GETXATTR]))(
+                frame, this, loc, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_getxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_getxattr_cbk, xl, xl->fops->getxattr,
+                    loc, name, xdata);
+}
+
+
+void
+unwind_getxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict,
+                 dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+
+}
+
+
+void
+set_getxattr_fop (long py_this, fop_getxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_GETXATTR] = (long)fop;
+}
+
+
+void
+set_getxattr_cbk (long py_this, fop_getxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_GETXATTR] = (long)cbk;
+}
+
+/* FOP: FSETXATTR */
+
+int32_t
+glupy_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FSETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fsetxattr_cbk_t)(priv->cbks[GLUPY_FSETXATTR]))(
+                frame, cookie, this, op_ret, op_errno,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FSETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fsetxattr_t)(priv->fops[GLUPY_FSETXATTR]))(
+                frame, this, fd, dict, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_fsetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fsetxattr_cbk, xl, xl->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+}
+
+
+void
+unwind_fsetxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_fsetxattr_fop (long py_this, fop_fsetxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FSETXATTR] = (long)fop;
+}
+
+void
+set_fsetxattr_cbk (long py_this, fop_fsetxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FSETXATTR] = (long)cbk;
+}
+
+/* FOP: FGETXATTR */
+
+int32_t
+glupy_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FGETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fgetxattr_cbk_t)(priv->cbks[GLUPY_FGETXATTR]))(
+                frame, cookie, this, op_ret, op_errno, dict,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FGETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fgetxattr_t)(priv->fops[GLUPY_FGETXATTR]))(
+                frame, this, fd, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_fgetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fgetxattr_cbk, xl, xl->fops->fgetxattr,
+                    fd, name, xdata);
+}
+
+
+void
+unwind_fgetxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict,
+                  dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+
+}
+
+
+void
+set_fgetxattr_fop (long py_this, fop_fgetxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FGETXATTR] = (long)fop;
+}
+
+
+void
+set_fgetxattr_cbk (long py_this, fop_fgetxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FGETXATTR] = (long)cbk;
+}
+
+/* FOP:REMOVEXATTR */
+
+int32_t
+glupy_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_REMOVEXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_removexattr_cbk_t)(priv->cbks[GLUPY_REMOVEXATTR]))(
+                frame, cookie, this, op_ret, op_errno, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_REMOVEXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_removexattr_t)(priv->fops[GLUPY_REMOVEXATTR]))(
+                frame, this, loc, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_removexattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_removexattr_cbk, xl, xl->fops->removexattr,
+                    loc, name, xdata);
+}
+
+
+void
+unwind_removexattr (call_frame_t *frame, long cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_removexattr_fop (long py_this, fop_removexattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_REMOVEXATTR] = (long)fop;
+}
+
+void
+set_removexattr_cbk (long py_this, fop_removexattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_REMOVEXATTR] = (long)cbk;
+}
+
+
+/* FOP:FREMOVEXATTR */
+
+int32_t
+glupy_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FREMOVEXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fremovexattr_cbk_t)(priv->cbks[GLUPY_FREMOVEXATTR]))(
+                frame, cookie, this, op_ret, op_errno, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FREMOVEXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fremovexattr_t)(priv->fops[GLUPY_FREMOVEXATTR]))(
+                frame, this, fd, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fremovexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_fremovexattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fremovexattr_cbk, xl, xl->fops->fremovexattr,
+                    fd, name, xdata);
+}
+
+
+void
+unwind_fremovexattr (call_frame_t *frame, long cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_fremovexattr_fop (long py_this, fop_fremovexattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FREMOVEXATTR] = (long)fop;
+}
+
+void
+set_fremovexattr_cbk (long py_this, fop_fremovexattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FREMOVEXATTR] = (long)cbk;
+}
+
+
+/* FOP: LINK*/
+int32_t
+glupy_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_LINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_link_cbk_t)(priv->cbks[GLUPY_LINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_LINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_link_t)(priv->fops[GLUPY_LINK]))(
+                frame, this, oldloc, newloc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_link_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc,
+                    xdata);
+        return 0;
+}
+
+void
+wind_link (call_frame_t *frame, xlator_t *xl, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_link_cbk, xl, xl->fops->link,
+                    oldloc, newloc, xdata);
+}
+
+void
+unwind_link (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, inode_t *inode,
+             struct iatt *buf, struct iatt *preparent,
+             struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_link_fop (long py_this, fop_link_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_LINK] = (long)fop;
+}
+
+void
+set_link_cbk (long py_this, fop_link_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_LINK] = (long)cbk;
+}
+
+/* FOP: SYMLINK*/
+int32_t
+glupy_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_SYMLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_symlink_cbk_t)(priv->cbks[GLUPY_SYMLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_SYMLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_symlink_t)(priv->fops[GLUPY_SYMLINK]))(
+                frame, this, linkname, loc, umask, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_symlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->symlink, linkname, loc,
+                    umask, xdata);
+        return 0;
+}
+
+void
+wind_symlink (call_frame_t *frame, xlator_t *xl, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_symlink_cbk, xl, xl->fops->symlink,
+                    linkname, loc, umask, xdata);
+}
+
+void
+unwind_symlink (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_symlink_fop (long py_this, fop_symlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_SYMLINK] = (long)fop;
+}
+
+void
+set_symlink_cbk (long py_this, fop_symlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_SYMLINK] = (long)cbk;
+}
+
+
+/* FOP: READLINK */
+int32_t
+glupy_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, const char *path,
+                    struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readlink_cbk_t)(priv->cbks[GLUPY_READLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                path, buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path,
+                             buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                size_t size, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readlink_t)(priv->fops[GLUPY_READLINK]))(
+                frame, this, loc, size, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readlink, loc,
+                    size, xdata);
+        return 0;
+}
+
+void
+wind_readlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               size_t size, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_readlink_cbk, xl, xl->fops->readlink,
+                    loc, size, xdata);
+}
+
+void
+unwind_readlink (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *path,
+                 struct iatt *buf, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf,
+                             xdata);
+}
+
+void
+set_readlink_fop (long py_this, fop_readlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READLINK] = (long)fop;
+}
+
+void
+set_readlink_cbk (long py_this, fop_readlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READLINK] = (long)cbk;
+}
+
+
+/* FOP: UNLINK */
+
+int32_t
+glupy_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_UNLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_unlink_cbk_t)(priv->cbks[GLUPY_UNLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              int xflags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_UNLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_unlink_t)(priv->fops[GLUPY_UNLINK]))(
+                frame, this, loc, xflags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc,
+                    xflags, xdata);
+        return 0;
+}
+
+void
+wind_unlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+             int xflags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_unlink_cbk, xl, xl->fops->unlink,
+                    loc, xflags, xdata);
+}
+
+void
+unwind_unlink (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iatt *preparent, struct iatt *postparent,
+               dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+}
+
+void
+set_unlink_fop (long py_this, fop_unlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_UNLINK] = (long)fop;
+}
+
+void
+set_unlink_cbk (long py_this, fop_unlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_UNLINK] = (long)cbk;
+}
+
+
+/* FOP: MKDIR */
+
+int32_t
+glupy_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_MKDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_mkdir_cbk_t)(priv->cbks[GLUPY_MKDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_MKDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_mkdir_t)(priv->fops[GLUPY_MKDIR]))(
+                frame, this, loc, mode, umask, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                    xdata);
+        return 0;
+}
+
+void
+wind_mkdir (call_frame_t *frame, xlator_t *xl, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
+{
+
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_mkdir_cbk, xl, xl->fops->mkdir,
+                    loc, mode, umask, xdata);
+}
+
+void
+unwind_mkdir (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, inode_t *inode,
+              struct iatt *buf, struct iatt *preparent,
+              struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_mkdir_fop (long py_this, fop_mkdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_MKDIR] = (long)fop;
+}
+
+void
+set_mkdir_cbk (long py_this, fop_mkdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_MKDIR] = (long)cbk;
+}
+
+
+/* FOP: RMDIR */
+
+int32_t
+glupy_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_RMDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_rmdir_cbk_t)(priv->cbks[GLUPY_RMDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             int xflags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_RMDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_rmdir_t)(priv->fops[GLUPY_RMDIR]))(
+                frame, this, loc, xflags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_rmdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rmdir, loc,
+                    xflags, xdata);
+        return 0;
+}
+
+void
+wind_rmdir (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+            int xflags, dict_t *xdata)
+{
+
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_rmdir_cbk, xl, xl->fops->rmdir,
+                    loc, xflags, xdata);
+}
+
+void
+unwind_rmdir (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno,
+              struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+}
+
+void
+set_rmdir_fop (long py_this, fop_rmdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_RMDIR] = (long)fop;
+}
+
+void
+set_rmdir_cbk (long py_this, fop_rmdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_RMDIR] = (long)cbk;
+}
+
+
+/* NON-FOP-SPECIFIC CODE */
+
+
+long
+get_id (call_frame_t *frame)
+{
+        return (long)(frame->local);
+}
+
+uint64_t
+get_rootunique (call_frame_t *frame)
+{
+        return frame->root->unique;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        glupy_private_t         *priv           = NULL;
+        char                    *module_name    = NULL;
+        PyObject                *py_mod_name    = NULL;
+        PyObject                *py_init_func   = NULL;
+        PyObject                *py_args        = NULL;
+        PyObject                *syspath        = NULL;
+	PyObject                *path           = NULL;
+	static gf_boolean_t      py_inited      = _gf_false;
+        void *                   err_cleanup    = &&err_return;
+
+        if (dict_get_str(this->options,"module-name",&module_name) != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "missing module-name");
+                return -1;
+        }
+
+	priv = GF_CALLOC (1, sizeof (glupy_private_t), gf_glupy_mt_priv);
+        if (!priv) {
+                goto *err_cleanup;
+        }
+        this->private = priv;
+        err_cleanup = &&err_free_priv;
+
+	if (!py_inited) {
+                Py_Initialize();
+                PyEval_InitThreads();
+#if 0
+                (void)pthread_key_create(&gil_init_key,NULL);
+                (void)pthread_setspecific(gil_init_key,(void *)1);
+#endif
+                /* PyEval_InitThreads takes this "for" us.  No thanks. */
+                PyEval_ReleaseLock();
+                py_inited = _gf_true;
+        }
+
+	/* Adjust python's path */
+	syspath = PySys_GetObject("path");
+	path = PyString_FromString(GLUSTER_PYTHON_PATH);
+	PyList_Append(syspath, path);
+	Py_DECREF(path);
+
+	py_mod_name = PyString_FromString(module_name);
+        if (!py_mod_name) {
+                gf_log (this->name, GF_LOG_ERROR, "could not create name");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "py_mod_name = %s", module_name);
+        priv->py_module = PyImport_Import(py_mod_name);
+        Py_DECREF(py_mod_name);
+        if (!priv->py_module) {
+                gf_log (this->name, GF_LOG_ERROR, "Python import failed");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        gf_log (this->name, GF_LOG_INFO, "Import of %s succeeded", module_name);
+        err_cleanup = &&err_deref_module;
+
+        py_init_func = PyObject_GetAttrString(priv->py_module, "xlator");
+        if (!py_init_func || !PyCallable_Check(py_init_func)) {
+                gf_log (this->name, GF_LOG_ERROR, "missing init func");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        err_cleanup = &&err_deref_init;
+
+        py_args = PyTuple_New(1);
+        if (!py_args) {
+                gf_log (this->name, GF_LOG_ERROR, "could not create args");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        PyTuple_SetItem(py_args,0,PyLong_FromLong((long)this));
+
+        /* TBD: pass in list of children */
+        priv->py_xlator = PyObject_CallObject(py_init_func, py_args);
+        Py_DECREF(py_args);
+        if (!priv->py_xlator) {
+                gf_log (this->name, GF_LOG_ERROR, "Python init failed");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "init returned %p", priv->py_xlator);
+
+        return 0;
+
+err_deref_init:
+        Py_DECREF(py_init_func);
+err_deref_module:
+        Py_DECREF(priv->py_module);
+err_free_priv:
+        GF_FREE(priv);
+err_return:
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        glupy_private_t *priv = this->private;
+
+        if (!priv)
+                return;
+        Py_DECREF(priv->py_xlator);
+        Py_DECREF(priv->py_module);
+        this->private = NULL;
+        GF_FREE (priv);
+
+        return;
+}
+
+struct xlator_fops fops = {
+        .lookup       = glupy_lookup,
+        .create       = glupy_create,
+        .open         = glupy_open,
+        .readv        = glupy_readv,
+        .writev       = glupy_writev,
+        .opendir      = glupy_opendir,
+        .readdir      = glupy_readdir,
+        .stat         = glupy_stat,
+        .fstat        = glupy_fstat,
+        .setxattr     = glupy_setxattr,
+        .getxattr     = glupy_getxattr,
+        .fsetxattr    = glupy_fsetxattr,
+        .fgetxattr    = glupy_fgetxattr,
+        .removexattr  = glupy_removexattr,
+        .fremovexattr = glupy_fremovexattr,
+        .link         = glupy_link,
+        .unlink       = glupy_unlink,
+        .readlink     = glupy_readlink,
+        .symlink      = glupy_symlink,
+        .mkdir        = glupy_mkdir,
+        .rmdir        = glupy_rmdir,
+        .statfs       = glupy_statfs,
+        .readdirp     = glupy_readdirp
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/glupy/src/glupy.h b/xlators/features/glupy/src/glupy.h
new file mode 100644
index 000000000..8661fce88
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.h
@@ -0,0 +1,69 @@
+/*
+   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   This file is part of GlusterFS.
+
+   GlusterFS is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published
+   by the Free Software Foundation; either version 3 of the License,
+   or (at your option) any later version.
+
+   GlusterFS is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __GLUPY_H__
+#define __GLUPY_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include "mem-types.h"
+
+enum {
+        GLUPY_LOOKUP = 0,
+        GLUPY_CREATE,
+        GLUPY_OPEN,
+        GLUPY_READV,
+        GLUPY_WRITEV,
+        GLUPY_OPENDIR,
+        GLUPY_READDIR,
+        GLUPY_READDIRP,
+        GLUPY_STAT,
+        GLUPY_FSTAT,
+        GLUPY_STATFS,
+        GLUPY_SETXATTR,
+        GLUPY_GETXATTR,
+        GLUPY_FSETXATTR,
+        GLUPY_FGETXATTR,
+        GLUPY_REMOVEXATTR,
+        GLUPY_FREMOVEXATTR,
+        GLUPY_LINK,
+        GLUPY_UNLINK,
+        GLUPY_READLINK,
+        GLUPY_SYMLINK,
+        GLUPY_MKNOD,
+        GLUPY_MKDIR,
+        GLUPY_RMDIR,
+        GLUPY_N_FUNCS
+};
+
+typedef struct {
+        PyObject        *py_module;
+        PyObject        *py_xlator;
+        long            fops[GLUPY_N_FUNCS];
+        long            cbks[GLUPY_N_FUNCS];
+} glupy_private_t;
+
+enum gf_glupy_mem_types_ {
+        gf_glupy_mt_priv = gf_common_mt_end + 1,
+        gf_glupy_mt_end
+};
+
+#endif /* __GLUPY_H__ */
diff --git a/xlators/features/glupy/src/glupy.py b/xlators/features/glupy/src/glupy.py
new file mode 100644
index 000000000..a5daa77d3
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.py
@@ -0,0 +1,841 @@
+import sys
+from ctypes import *
+
+dl = CDLL("",RTLD_GLOBAL)
+
+
+class call_frame_t (Structure):
+        pass
+
+class dev_t (Structure):
+        pass
+
+
+class dict_t (Structure):
+        pass
+
+
+class gf_dirent_t (Structure):
+        pass
+
+
+class iobref_t (Structure):
+	pass
+
+
+class iovec_t (Structure):
+        pass
+
+
+class list_head (Structure):
+        pass
+
+list_head._fields_ = [
+                ("next", POINTER(list_head)),
+                ("prev", POINTER(list_head))
+        ]
+
+
+class rwxperm_t (Structure):
+        _fields_ = [
+                ("read", c_uint8, 1),
+                ("write", c_uint8, 1),
+                ("execn", c_uint8, 1)
+        ]
+
+
+class statvfs_t (Structure):
+        pass
+
+
+class xlator_t (Structure):
+        pass
+
+
+class ia_prot_t (Structure):
+        _fields_ = [
+                ("suid", c_uint8, 1),
+                ("sgid", c_uint8, 1),
+                ("sticky", c_uint8, 1),
+                ("owner", rwxperm_t),
+                ("group", rwxperm_t),
+                ("other", rwxperm_t)
+        ]
+
+# For checking file type.
+(IA_INVAL, IA_IFREG, IA_IFDIR, IA_IFLNK, IA_IFBLK, IA_IFCHR, IA_IFIFO,
+ IA_IFSOCK) = xrange(8)
+
+
+class iatt_t (Structure):
+        _fields_ = [
+                ("ia_no", c_uint64),
+                ("ia_gfid", c_ubyte * 16),
+                ("ia_dev", c_uint64),
+                ("ia_type", c_uint),
+                ("ia_prot", ia_prot_t),
+                ("ia_nlink", c_uint32),
+                ("ia_uid", c_uint32),
+                ("ia_gid", c_uint32),
+                ("ia_rdev", c_uint64),
+                ("ia_size", c_uint64),
+                ("ia_blksize", c_uint32),
+                ("ia_blocks", c_uint64),
+                ("ia_atime", c_uint32 ),
+                ("ia_atime_nsec", c_uint32),
+                ("ia_mtime", c_uint32),
+                ("ia_mtime_nsec", c_uint32),
+                ("ia_ctime", c_uint32),
+                ("ia_ctime_nsec", c_uint32)
+        ]
+
+
+class mem_pool (Structure):
+        _fields_ = [
+                ("list", list_head),
+                ("hot_count", c_int),
+                ("cold_count", c_int),
+                ("lock", c_void_p),
+                ("padded_sizeof_type", c_ulong),
+                ("pool", c_void_p),
+                ("pool_end", c_void_p),
+                ("real_sizeof_type", c_int),
+                ("alloc_count", c_uint64),
+                ("pool_misses", c_uint64),
+                ("max_alloc", c_int),
+                ("curr_stdalloc", c_int),
+                ("max_stdalloc", c_int),
+                ("name", c_char_p),
+                ("global_list", list_head)
+        ]
+
+
+class U_ctx_key_inode (Union):
+        _fields_ = [
+                ("key", c_uint64),
+                ("xl_key", POINTER(xlator_t))
+        ]
+
+
+class U_ctx_value1 (Union):
+        _fields_ = [
+                ("value1", c_uint64),
+                ("ptr1", c_void_p)
+        ]
+
+
+class U_ctx_value2 (Union):
+        _fields_ = [
+                ("value2", c_uint64),
+                ("ptr2", c_void_p)
+        ]
+
+class inode_ctx (Structure):
+        _anonymous_ = ("u_key","u_value1","u_value2",)
+        _fields_ = [
+                ("u_key", U_ctx_key_inode),
+                ("u_value1", U_ctx_value1),
+                ("u_value2", U_ctx_value2)
+        ]
+
+class inode_t (Structure):
+        pass
+
+class inode_table_t (Structure):
+        _fields_ = [
+                ("lock", c_void_p),
+                ("hashsize", c_size_t),
+                ("name", c_char_p),
+                ("root", POINTER(inode_t)),
+                ("xl", POINTER(xlator_t)),
+                ("lru_limit", c_uint32),
+                ("inode_hash", POINTER(list_head)),
+                ("name_hash", POINTER(list_head)),
+                ("active", list_head),
+                ("active_size", c_uint32),
+                ("lru", list_head),
+                ("lru_size", c_uint32),
+                ("purge", list_head),
+                ("purge_size", c_uint32),
+                ("inode_pool", POINTER(mem_pool)),
+                ("dentry_pool", POINTER(mem_pool)),
+                ("fd_mem_pool", POINTER(mem_pool))
+        ]
+
+inode_t._fields_ = [
+                ("table", POINTER(inode_table_t)),
+                ("gfid", c_ubyte * 16),
+                ("lock", c_void_p),
+                ("nlookup", c_uint64),
+                ("fd_count", c_uint32),
+                ("ref", c_uint32),
+                ("ia_type", c_uint),
+                ("fd_list", list_head),
+                ("dentry_list", list_head),
+                ("hashv", list_head),
+                ("listv", list_head),
+                ("ctx", POINTER(inode_ctx))
+        ]
+
+
+
+class U_ctx_key_fd (Union):
+        _fields_ = [
+                ("key", c_uint64),
+                ("xl_key", c_void_p)
+        ]
+
+class fd_lk_ctx (Structure):
+        _fields_ = [
+                ("lk_list", list_head),
+                ("ref", c_int),
+                ("lock", c_void_p)
+        ]
+
+class fd_ctx (Structure):
+        _anonymous_ = ("u_key","u_value1")
+        _fields_ = [
+                ("u_key", U_ctx_key_fd),
+                ("u_value1", U_ctx_value1)
+        ]
+
+class fd_t (Structure):
+        _fields_ = [
+                ("pid", c_uint64),
+                ("flags", c_int32),
+                ("refcount", c_int32),
+                ("inode_list", list_head),
+                ("inode", POINTER(inode_t)),
+                ("lock", c_void_p),
+                ("ctx", POINTER(fd_ctx)),
+                ("xl_count", c_int),
+                ("lk_ctx", POINTER(fd_lk_ctx)),
+                ("anonymous", c_uint)
+        ]
+
+class loc_t (Structure):
+        _fields_ = [
+                ("path", c_char_p),
+                ("name", c_char_p),
+                ("inode", POINTER(inode_t)),
+                ("parent", POINTER(inode_t)),
+                ("gfid", c_ubyte * 16),
+                ("pargfid", c_ubyte * 16),
+        ]
+
+
+
+def _init_op (a_class, fop, cbk, wind, unwind):
+        # Decorators, used by translators. We could pass the signatures as
+        # parameters, but it's actually kind of nice to keep them around for
+        # inspection.
+        a_class.fop_type = apply(CFUNCTYPE,a_class.fop_sig)
+        a_class.cbk_type = apply(CFUNCTYPE,a_class.cbk_sig)
+        # Dispatch-function registration.
+        fop.restype = None
+        fop.argtypes = [ c_long, a_class.fop_type ]
+        # Callback-function registration.
+        cbk.restype = None
+        cbk.argtypes = [ c_long, a_class.cbk_type ]
+        # STACK_WIND function.
+        wind.restype = None
+        wind.argtypes = list(a_class.fop_sig[1:])
+        # STACK_UNWIND function.
+        unwind.restype = None
+        unwind.argtypes = list(a_class.cbk_sig[1:])
+
+class OpLookup:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(dict_t), POINTER(iatt_t))
+_init_op (OpLookup, dl.set_lookup_fop, dl.set_lookup_cbk,
+                    dl.wind_lookup,    dl.unwind_lookup)
+
+class OpCreate:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, c_uint, c_uint, POINTER(fd_t),
+                   POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(inode_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpCreate, dl.set_create_fop, dl.set_create_cbk,
+                    dl.wind_create,    dl.unwind_create)
+
+class OpOpen:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(fd_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
+_init_op (OpOpen, dl.set_open_fop, dl.set_open_cbk,
+                  dl.wind_open,    dl.unwind_open)
+
+class OpReadv:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, c_uint32, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iovec_t), c_int, POINTER(iatt_t),
+                   POINTER(iobref_t), POINTER(dict_t))
+_init_op (OpReadv, dl.set_readv_fop, dl.set_readv_cbk,
+                   dl.wind_readv,    dl.unwind_readv)
+class OpWritev:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(iovec_t), c_int, c_long, c_uint32,
+                   POINTER(iobref_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpWritev, dl.set_writev_fop, dl.set_writev_cbk,
+                    dl.wind_writev,    dl.unwind_writev)
+
+class OpOpendir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(fd_t) ,POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
+_init_op (OpOpendir, dl.set_opendir_fop, dl.set_opendir_cbk,
+                     dl.wind_opendir,    dl.unwind_opendir)
+
+class OpReaddir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
+_init_op (OpReaddir, dl.set_readdir_fop, dl.set_readdir_cbk,
+                     dl.wind_readdir,    dl.unwind_readdir)
+
+class OpReaddirp:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
+_init_op (OpReaddirp, dl.set_readdirp_fop, dl.set_readdirp_cbk,
+                      dl.wind_readdirp,    dl.unwind_readdirp)
+
+class OpStat:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpStat, dl.set_stat_fop, dl.set_stat_cbk,
+                  dl.wind_stat,    dl.unwind_stat)
+
+class OpFstat:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpFstat, dl.set_fstat_fop, dl.set_fstat_cbk,
+                   dl.wind_fstat,    dl.unwind_fstat)
+
+class OpStatfs:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(statvfs_t), POINTER(dict_t))
+_init_op (OpStatfs, dl.set_statfs_fop, dl.set_statfs_cbk,
+                    dl.wind_statfs,    dl.unwind_statfs)
+
+
+class OpSetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t), c_int32,
+                   POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpSetxattr, dl.set_setxattr_fop, dl.set_setxattr_cbk,
+                      dl.wind_setxattr,    dl.unwind_setxattr)
+
+class OpGetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_char_p, POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
+_init_op (OpGetxattr, dl.set_getxattr_fop, dl.set_getxattr_cbk,
+                      dl.wind_getxattr,    dl.unwind_getxattr)
+
+class OpFsetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(dict_t), c_int32,
+                   POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpFsetxattr, dl.set_fsetxattr_fop, dl.set_fsetxattr_cbk,
+                       dl.wind_fsetxattr,    dl.unwind_fsetxattr)
+
+class OpFgetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_char_p, POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
+_init_op (OpFgetxattr, dl.set_fgetxattr_fop, dl.set_fgetxattr_cbk,
+                       dl.wind_fgetxattr,    dl.unwind_fgetxattr)
+
+class OpRemovexattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_char_p, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpRemovexattr, dl.set_removexattr_fop, dl.set_removexattr_cbk,
+                         dl.wind_removexattr,    dl.unwind_removexattr)
+
+
+class OpFremovexattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_char_p, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpFremovexattr, dl.set_fremovexattr_fop, dl.set_fremovexattr_cbk,
+                          dl.wind_fremovexattr,    dl.unwind_fremovexattr)
+
+class OpLink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpLink, dl.set_link_fop, dl.set_link_cbk,
+                  dl.wind_link,    dl.unwind_link)
+
+class OpSymlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   c_char_p, POINTER(loc_t), c_uint, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpSymlink, dl.set_symlink_fop, dl.set_symlink_cbk,
+                     dl.wind_symlink,    dl.unwind_symlink)
+
+class OpUnlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpUnlink, dl.set_unlink_fop, dl.set_unlink_cbk,
+                    dl.wind_unlink,    dl.unwind_unlink)
+
+class OpReadlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_size_t, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, c_char_p, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpReadlink, dl.set_readlink_fop, dl.set_readlink_cbk,
+                      dl.wind_readlink,    dl.unwind_readlink)
+
+class OpMkdir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_uint, c_uint, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpMkdir, dl.set_mkdir_fop, dl.set_mkdir_cbk,
+                   dl.wind_mkdir,    dl.unwind_mkdir)
+
+class OpRmdir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpRmdir, dl.set_rmdir_fop, dl.set_rmdir_cbk,
+                   dl.wind_rmdir,    dl.unwind_rmdir)
+
+
+class Translator:
+        def __init__ (self, c_this):
+                # This is only here to keep references to the stubs we create,
+                # because ctypes doesn't and glupy.so can't because it doesn't
+                # get a pointer to the actual Python object. It's a dictionary
+                # instead of a list in case we ever allow changing fops/cbks
+                # after initialization and need to look them up.
+                self.stub_refs = {}
+                funcs = dir(self.__class__)
+                if "lookup_fop" in funcs:
+                        @OpLookup.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.lookup_fop (frame, this, loc, xdata)
+                        self.stub_refs["lookup_fop"] = stub
+                        dl.set_lookup_fop(c_this,stub)
+                if "lookup_cbk" in funcs:
+                        @OpLookup.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, inode,
+                                  buf, xdata, postparent, s=self):
+                                return s.lookup_cbk(frame, cookie, this, op_ret,
+                                                    op_errno, inode, buf, xdata,
+                                                    postparent)
+                        self.stub_refs["lookup_cbk"] = stub
+                        dl.set_lookup_cbk(c_this,stub)
+                if "create_fop" in funcs:
+                        @OpCreate.fop_type
+                        def stub (frame, this, loc, flags, mode, umask, fd,
+                                  xdata, s=self):
+                                return s.create_fop (frame, this, loc, flags,
+                                                     mode, umask, fd, xdata)
+                        self.stub_refs["create_fop"] = stub
+                        dl.set_create_fop(c_this,stub)
+                if "create_cbk" in funcs:
+                        @OpCreate.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.create_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, fd,
+                                                     inode, buf, preparent,
+                                                     postparent, xdata)
+                        self.stub_refs["create_cbk"] = stub
+                        dl.set_create_cbk(c_this,stub)
+                if "open_fop" in funcs:
+                        @OpOpen.fop_type
+                        def stub (frame, this, loc, flags, fd,
+                                  xdata, s=self):
+                                return s.open_fop (frame, this, loc, flags,
+                                                   fd, xdata)
+                        self.stub_refs["open_fop"] = stub
+                        dl.set_open_fop(c_this,stub)
+                if "open_cbk" in funcs:
+                        @OpOpen.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  xdata, s=self):
+                                return s.open_cbk (frame, cookie, this,
+                                                   op_ret, op_errno, fd,
+                                                   xdata)
+                        self.stub_refs["open_cbk"] = stub
+                        dl.set_open_cbk(c_this,stub)
+                if "readv_fop" in funcs:
+                        @OpReadv.fop_type
+                        def stub (frame, this, fd, size, offset, flags,
+                                  xdata, s=self):
+                                return s.readv_fop (frame, this, fd, size,
+                                                    offset, flags, xdata)
+                        self.stub_refs["readv_fop"] = stub
+                        dl.set_readv_fop(c_this,stub)
+                if "readv_cbk" in funcs:
+                        @OpReadv.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  vector, count, stbuf, iobref, xdata,
+                                  s=self):
+                                return s.readv_cbk (frame, cookie, this,
+                                                    op_ret, op_errno, vector,
+                                                    count, stbuf, iobref,
+                                                    xdata)
+                        self.stub_refs["readv_cbk"] = stub
+                        dl.set_readv_cbk(c_this,stub)
+                if "writev_fop" in funcs:
+                        @OpWritev.fop_type
+                        def stub (frame, this, fd, vector, count,
+                                  offset, flags, iobref, xdata, s=self):
+                                return s.writev_fop (frame, this, fd, vector,
+                                                     count, offset, flags,
+                                                     iobref, xdata)
+                        self.stub_refs["writev_fop"] = stub
+                        dl.set_writev_fop(c_this,stub)
+                if "writev_cbk" in funcs:
+                        @OpWritev.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  prebuf, postbuf, xdata, s=self):
+                                return s.writev_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, prebuf,
+                                                     postbuf, xdata)
+                        self.stub_refs["writev_cbk"] = stub
+                        dl.set_writev_cbk(c_this,stub)
+                if "opendir_fop" in funcs:
+                        @OpOpendir.fop_type
+                        def stub (frame, this, loc, fd, xdata, s=self):
+                                return s.opendir_fop (frame, this, loc, fd,
+                                                      xdata)
+                        self.stub_refs["opendir_fop"] = stub
+                        dl.set_opendir_fop(c_this,stub)
+                if "opendir_cbk" in funcs:
+                        @OpOpendir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  xdata, s=self):
+                                return s.opendir_cbk(frame, cookie, this,
+                                                     op_ret, op_errno, fd,
+                                                     xdata)
+                        self.stub_refs["opendir_cbk"] = stub
+                        dl.set_opendir_cbk(c_this,stub)
+                if "readdir_fop" in funcs:
+                        @OpReaddir.fop_type
+                        def stub (frame, this, fd, size, offset, xdata, s=self):
+                                return s.readdir_fop (frame, this, fd, size,
+                                                      offset, xdata)
+                        self.stub_refs["readdir_fop"] = stub
+                        dl.set_readdir_fop(c_this,stub)
+                if "readdir_cbk" in funcs:
+                        @OpReaddir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  entries, xdata, s=self):
+                                return s.readdir_cbk(frame, cookie, this,
+                                                     op_ret, op_errno, entries,
+                                                     xdata)
+                        self.stub_refs["readdir_cbk"] = stub
+                        dl.set_readdir_cbk(c_this,stub)
+                if "readdirp_fop" in funcs:
+                        @OpReaddirp.fop_type
+                        def stub (frame, this, fd, size, offset, xdata, s=self):
+                                return s.readdirp_fop (frame, this, fd, size,
+                                                       offset, xdata)
+                        self.stub_refs["readdirp_fop"] = stub
+                        dl.set_readdirp_fop(c_this,stub)
+                if "readdirp_cbk" in funcs:
+                        @OpReaddirp.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  entries, xdata, s=self):
+                                return s.readdirp_cbk (frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       entries, xdata)
+                        self.stub_refs["readdirp_cbk"] = stub
+                        dl.set_readdirp_cbk(c_this,stub)
+                if "stat_fop" in funcs:
+                        @OpStat.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.stat_fop (frame, this, loc, xdata)
+                        self.stub_refs["stat_fop"] = stub
+                        dl.set_stat_fop(c_this,stub)
+                if "stat_cbk" in funcs:
+                        @OpStat.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.stat_cbk(frame, cookie, this, op_ret,
+                                                  op_errno, buf, xdata)
+                        self.stub_refs["stat_cbk"] = stub
+                        dl.set_stat_cbk(c_this,stub)
+                if "fstat_fop" in funcs:
+                        @OpFstat.fop_type
+                        def stub (frame, this, fd, xdata, s=self):
+                                return s.fstat_fop (frame, this, fd, xdata)
+                        self.stub_refs["fstat_fop"] = stub
+                        dl.set_fstat_fop(c_this,stub)
+                if "fstat_cbk" in funcs:
+                        @OpFstat.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.fstat_cbk(frame, cookie, this, op_ret,
+                                                   op_errno, buf, xdata)
+                        self.stub_refs["fstat_cbk"] = stub
+                        dl.set_fstat_cbk(c_this,stub)
+                if "statfs_fop" in funcs:
+                        @OpStatfs.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.statfs_fop (frame, this, loc, xdata)
+                        self.stub_refs["statfs_fop"] = stub
+                        dl.set_statfs_fop(c_this,stub)
+                if "statfs_cbk" in funcs:
+                        @OpStatfs.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.statfs_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, buf,
+                                                     xdata)
+                        self.stub_refs["statfs_cbk"] = stub
+                        dl.set_statfs_cbk(c_this,stub)
+                if "setxattr_fop" in funcs:
+                        @OpSetxattr.fop_type
+                        def stub (frame, this, loc, dictionary, flags, xdata,
+                                  s=self):
+                                return s.setxattr_fop (frame, this, loc,
+                                                       dictionary, flags,
+                                                       xdata)
+                        self.stub_refs["setxattr_fop"] = stub
+                        dl.set_setxattr_fop(c_this,stub)
+                if "setxattr_cbk" in funcs:
+                        @OpSetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
+                                  s=self):
+                                return s.setxattr_cbk(frame, cookie, this,
+                                                      op_ret, op_errno, xdata)
+                        self.stub_refs["setxattr_cbk"] = stub
+                        dl.set_setxattr_cbk(c_this,stub)
+                if "getxattr_fop" in funcs:
+                        @OpGetxattr.fop_type
+                        def stub (frame, this, loc, name, xdata, s=self):
+                                return s.getxattr_fop (frame, this, loc, name,
+                                                       xdata)
+                        self.stub_refs["getxattr_fop"] = stub
+                        dl.set_getxattr_fop(c_this,stub)
+                if "getxattr_cbk" in funcs:
+                        @OpGetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  dictionary, xdata, s=self):
+                                return s.getxattr_cbk(frame, cookie, this,
+                                                      op_ret, op_errno,
+                                                      dictionary, xdata)
+                        self.stub_refs["getxattr_cbk"] = stub
+                        dl.set_getxattr_cbk(c_this,stub)
+                if "fsetxattr_fop" in funcs:
+                        @OpFsetxattr.fop_type
+                        def stub (frame, this, fd, dictionary, flags, xdata,
+                                  s=self):
+                                return s.fsetxattr_fop (frame, this, fd,
+                                                        dictionary, flags,
+                                                        xdata)
+                        self.stub_refs["fsetxattr_fop"] = stub
+                        dl.set_fsetxattr_fop(c_this,stub)
+                if "fsetxattr_cbk" in funcs:
+                        @OpFsetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
+                                  s=self):
+                                return s.fsetxattr_cbk(frame, cookie, this,
+                                                       op_ret, op_errno, xdata)
+                        self.stub_refs["fsetxattr_cbk"] = stub
+                        dl.set_fsetxattr_cbk(c_this,stub)
+                if "fgetxattr_fop" in funcs:
+                        @OpFgetxattr.fop_type
+                        def stub (frame, this, fd, name, xdata, s=self):
+                                return s.fgetxattr_fop (frame, this, fd, name,
+                                                        xdata)
+                        self.stub_refs["fgetxattr_fop"] = stub
+                        dl.set_fgetxattr_fop(c_this,stub)
+                if "fgetxattr_cbk" in funcs:
+                        @OpFgetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  dictionary, xdata, s=self):
+                                return s.fgetxattr_cbk(frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       dictionary, xdata)
+                        self.stub_refs["fgetxattr_cbk"] = stub
+                        dl.set_fgetxattr_cbk(c_this,stub)
+                if "removexattr_fop" in funcs:
+                        @OpRemovexattr.fop_type
+                        def stub (frame, this, loc, name, xdata, s=self):
+                                return s.removexattr_fop (frame, this, loc,
+                                                          name, xdata)
+                        self.stub_refs["removexattr_fop"] = stub
+                        dl.set_removexattr_fop(c_this,stub)
+                if "removexattr_cbk" in funcs:
+                        @OpRemovexattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  xdata, s=self):
+                                return s.removexattr_cbk(frame, cookie, this,
+                                                         op_ret, op_errno,
+                                                         xdata)
+                        self.stub_refs["removexattr_cbk"] = stub
+                        dl.set_removexattr_cbk(c_this,stub)
+                if "fremovexattr_fop" in funcs:
+                        @OpFremovexattr.fop_type
+                        def stub (frame, this, fd, name, xdata, s=self):
+                                return s.fremovexattr_fop (frame, this, fd,
+                                                           name, xdata)
+                        self.stub_refs["fremovexattr_fop"] = stub
+                        dl.set_fremovexattr_fop(c_this,stub)
+                if "fremovexattr_cbk" in funcs:
+                        @OpFremovexattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  xdata, s=self):
+                                return s.fremovexattr_cbk(frame, cookie, this,
+                                                          op_ret, op_errno,
+                                                          xdata)
+                        self.stub_refs["fremovexattr_cbk"] = stub
+                        dl.set_fremovexattr_cbk(c_this,stub)
+                if "link_fop" in funcs:
+                        @OpLink.fop_type
+                        def stub (frame, this, oldloc, newloc,
+                                  xdata, s=self):
+                                return s.link_fop (frame, this, oldloc,
+                                                   newloc, xdata)
+                        self.stub_refs["link_fop"] = stub
+                        dl.set_link_fop(c_this,stub)
+                if "link_cbk" in funcs:
+                        @OpLink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.link_cbk (frame, cookie, this,
+                                                   op_ret, op_errno, inode,
+                                                   buf, preparent,
+                                                   postparent, xdata)
+                        self.stub_refs["link_cbk"] = stub
+                        dl.set_link_cbk(c_this,stub)
+                if "symlink_fop" in funcs:
+                        @OpSymlink.fop_type
+                        def stub (frame, this, linkname, loc,
+                                  umask, xdata, s=self):
+                                return s.symlink_fop (frame, this, linkname,
+                                                      loc, umask, xdata)
+                        self.stub_refs["symlink_fop"] = stub
+                        dl.set_symlink_fop(c_this,stub)
+                if "symlink_cbk" in funcs:
+                        @OpSymlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.symlink_cbk (frame, cookie, this,
+                                                      op_ret, op_errno, inode,
+                                                      buf, preparent,
+                                                      postparent, xdata)
+                        self.stub_refs["symlink_cbk"] = stub
+                        dl.set_symlink_cbk(c_this,stub)
+                if "unlink_fop" in funcs:
+                        @OpUnlink.fop_type
+                        def stub (frame, this, loc, xflags,
+                                  xdata, s=self):
+                                return s.unlink_fop (frame, this, loc,
+                                                     xflags, xdata)
+                        self.stub_refs["unlink_fop"] = stub
+                        dl.set_unlink_fop(c_this,stub)
+                if "unlink_cbk" in funcs:
+                        @OpUnlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  preparent, postparent, xdata, s=self):
+                                return s.unlink_cbk (frame, cookie, this,
+                                                     op_ret, op_errno,
+                                                     preparent, postparent,
+                                                     xdata)
+                        self.stub_refs["unlink_cbk"] = stub
+                        dl.set_unlink_cbk(c_this,stub)
+                if "readlink_fop" in funcs:
+                        @OpReadlink.fop_type
+                        def stub (frame, this, loc, size,
+                                  xdata, s=self):
+                                return s.readlink_fop (frame, this, loc,
+                                                       size, xdata)
+                        self.stub_refs["readlink_fop"] = stub
+                        dl.set_readlink_fop(c_this,stub)
+                if "readlink_cbk" in funcs:
+                        @OpReadlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  path, buf, xdata, s=self):
+                                return s.readlink_cbk (frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       path, buf, xdata)
+                        self.stub_refs["readlink_cbk"] = stub
+                        dl.set_readlink_cbk(c_this,stub)
+                if "mkdir_fop" in funcs:
+                        @OpMkdir.fop_type
+                        def stub (frame, this, loc, mode, umask, xdata,
+                                  s=self):
+                                return s.mkdir_fop (frame, this, loc, mode,
+                                                    umask, xdata)
+                        self.stub_refs["mkdir_fop"] = stub
+                        dl.set_mkdir_fop(c_this,stub)
+                if "mkdir_cbk" in funcs:
+                        @OpMkdir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, inode,
+                                  buf, preparent, postparent, xdata, s=self):
+                                return s.mkdir_cbk (frame, cookie, this,
+                                                    op_ret, op_errno, inode,
+                                                    buf, preparent,
+                                                    postparent, xdata)
+                        self.stub_refs["mkdir_cbk"] = stub
+                        dl.set_mkdir_cbk(c_this,stub)
+                if "rmdir_fop" in funcs:
+                        @OpRmdir.fop_type
+                        def stub (frame, this, loc, xflags,
+                                  xdata, s=self):
+                                return s.rmdir_fop (frame, this, loc,
+                                                    xflags, xdata)
+                        self.stub_refs["rmdir_fop"] = stub
+                        dl.set_rmdir_fop(c_this,stub)
+                if "rmdir_cbk" in funcs:
+                        @OpRmdir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  preparent, postparent, xdata, s=self):
+                                return s.rmdir_cbk (frame, cookie, this,
+                                                    op_ret, op_errno,
+                                                    preparent, postparent,
+                                                    xdata)
+                        self.stub_refs["rmdir_cbk"] = stub
+                        dl.set_rmdir_cbk(c_this,stub)
diff --git a/xlators/features/glupy/src/setup.py.in b/xlators/features/glupy/src/setup.py.in
new file mode 100644
index 000000000..1aea9875f
--- /dev/null
+++ b/xlators/features/glupy/src/setup.py.in
@@ -0,0 +1,24 @@
+from distutils.core import setup
+
+DESC = """GlusterFS is a clustered file-system capable of scaling to
+several petabytes. It aggregates various storage bricks over Infiniband
+RDMA or TCP/IP interconnect into one large parallel network file system.
+GlusterFS is one of the most sophisticated file systems in terms of
+features and extensibility.  It borrows a powerful concept called
+Translators from GNU Hurd kernel. Much of the code in GlusterFS is in
+user space and easily manageable.
+
+This package contains Glupy, the Python translator interface for GlusterFS."""
+
+setup(
+    name='glusterfs-glupy',
+    version='@PACKAGE_VERSION@',
+    description='Glupy is the Python translator interface for GlusterFS',
+    long_description=DESC,
+    author='Gluster Community',
+    author_email='gluster-devel@nongnu.org',
+    license='LGPLv3',
+    url='http://gluster.org/',
+    package_dir={'gluster':''},
+    packages=['gluster']
+)
diff --git a/xlators/features/index/src/Makefile.am b/xlators/features/index/src/Makefile.am
index 5d037c7ec..73bb8972e 100644
--- a/xlators/features/index/src/Makefile.am
+++ b/xlators/features/index/src/Makefile.am
@@ -1,15 +1,17 @@
 xlator_LTLIBRARIES = index.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-index_la_LDFLAGS = -module -avoidversion
+index_la_LDFLAGS = -module -avoid-version
 
 index_la_SOURCES = index.c
 index_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = index.h index-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src -shared -nostartfiles $(GF_CFLAGS)
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/features/index/src/index-mem-types.h b/xlators/features/index/src/index-mem-types.h
index bd10f518f..553d492df 100644
--- a/xlators/features/index/src/index-mem-types.h
+++ b/xlators/features/index/src/index-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2012-2012 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __QUIESCE_MEM_TYPES_H__
 #define __QUIESCE_MEM_TYPES_H__
 
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index 12309b83f..5c1c65fbd 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2012-2012 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -25,6 +15,7 @@
 #include "index.h"
 #include "options.h"
 #include "glusterfs3-xdr.h"
+#include "syscall.h"
 
 #define XATTROP_SUBDIR "xattrop"
 
@@ -371,7 +362,7 @@ index_add (xlator_t *this, uuid_t gfid, const char *subdir)
         index_get_index (priv, index);
         make_index_path (priv->index_basepath, subdir,
                          index, index_path, sizeof (index_path));
-        ret = link (index_path, gfid_path);
+        ret = sys_link (index_path, gfid_path);
         if (!ret || (errno == EEXIST))  {
                 ret = 0;
                 goto out;
@@ -402,7 +393,7 @@ index_add (xlator_t *this, uuid_t gfid, const char *subdir)
         if (fd >= 0)
                 close (fd);
 
-        ret = link (index_path, gfid_path);
+        ret = sys_link (index_path, gfid_path);
         if (ret && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
                         "add to index (%s)", uuid_utoa (gfid),
@@ -441,27 +432,27 @@ out:
         return ret;
 }
 
+static int
+_check_key_is_zero_filled (dict_t *d, char *k, data_t *v,
+                           void *tmp)
+{
+        if (mem_0filled ((const char*)v->data, v->len)) {
+                /* -1 means, no more iterations, treat as 'break' */
+                return -1;
+        }
+        return 0;
+}
+
 void
-_xattrop_index_action (xlator_t *this, inode_t *inode,  dict_t *xattr)
+_index_action (xlator_t *this, inode_t *inode, gf_boolean_t zero_xattr)
 {
-        data_pair_t       *trav = NULL;
-        gf_boolean_t      zero_xattr = _gf_true;
+        int               ret  = 0;
         index_inode_ctx_t *ctx = NULL;
-        int               ret = 0;
 
-        trav = xattr->members_list;
-        while (trav && inode) {
-                if (mem_0filled ((const char*)trav->value->data,
-                                 trav->value->len)) {
-                        zero_xattr = _gf_false;
-                        break;
-                }
-                trav = trav->next;
-        }
         ret = index_inode_ctx_get (inode, this, &ctx);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Not able to %s %s -> index",
-                        zero_xattr?"add":"del", uuid_utoa (inode->gfid));
+                        zero_xattr?"del":"add", uuid_utoa (inode->gfid));
                 goto out;
         }
         if (zero_xattr) {
@@ -482,6 +473,19 @@ out:
 }
 
 void
+_xattrop_index_action (xlator_t *this, inode_t *inode,  dict_t *xattr)
+{
+        gf_boolean_t      zero_xattr = _gf_true;
+        int               ret = 0;
+
+        ret = dict_foreach (xattr, _check_key_is_zero_filled, NULL);
+        if (ret == -1)
+                zero_xattr = _gf_false;
+        _index_action (this, inode, zero_xattr);
+        return;
+}
+
+void
 fop_xattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr)
 {
         _xattrop_index_action (this, inode, xattr);
@@ -605,10 +609,10 @@ unlock:
                 frame = new->frame;
                 if (new->fop == GF_FOP_XATTROP) {
                         INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM,
-                                            NULL);
+                                            NULL, NULL);
                 } else if (new->fop == GF_FOP_FXATTROP) {
                         INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM,
-                                            NULL);
+                                            NULL, NULL);
                 }
                 call_stub_destroy (new);
         } else if (stub) {
@@ -619,7 +623,7 @@ unlock:
 
 int32_t
 index_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                   int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
         inode_t *inode = NULL;
 
@@ -628,7 +632,7 @@ index_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         fop_xattrop_index_action (this, frame->local, xattr);
 out:
-        INDEX_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr);
+        INDEX_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr, xdata);
         index_queue_process (this, inode, NULL);
         inode_unref (inode);
 
@@ -637,7 +641,8 @@ out:
 
 int32_t
 index_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                    int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                    dict_t *xdata)
 {
         inode_t *inode = NULL;
 
@@ -647,7 +652,7 @@ index_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         fop_fxattrop_index_action (this, frame->local, xattr);
 out:
-        INDEX_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, xattr);
+        INDEX_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, xattr, xdata);
         index_queue_process (this, inode, NULL);
         inode_unref (inode);
 
@@ -656,25 +661,36 @@ out:
 
 int
 index_xattrop_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       gf_xattrop_flags_t optype, dict_t *xattr)
+                       gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
+        //In wind phase bring the gfid into index. This way if the brick crashes
+        //just after posix performs xattrop before _cbk reaches index xlator
+        //we will still have the gfid in index.
+        _index_action (this, frame->local, _gf_false);
+
         STACK_WIND (frame, index_xattrop_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr);
+                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr,
+                    xdata);
         return 0;
 }
 
 int
 index_fxattrop_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        gf_xattrop_flags_t optype, dict_t *xattr)
+                        gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
+        //In wind phase bring the gfid into index. This way if the brick crashes
+        //just after posix performs xattrop before _cbk reaches index xlator
+        //we will still have the gfid in index.
+        _index_action (this, frame->local, _gf_false);
         STACK_WIND (frame, index_fxattrop_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr);
+                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr,
+                    xdata);
         return 0;
 }
 
 int32_t
 index_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	       gf_xattrop_flags_t flags, dict_t *dict)
+	       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
 
@@ -683,9 +699,9 @@ index_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         frame->local = inode_ref (loc->inode);
         stub = fop_xattrop_stub (frame, index_xattrop_wrapper,
-                                 loc, flags, dict);
+                                 loc, flags, dict, xdata);
         if (!stub) {
-                INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM, NULL);
+                INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -693,13 +709,13 @@ index_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 out:
         STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict);
+                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
         return 0;
 }
 
 int32_t
 index_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		gf_xattrop_flags_t flags, dict_t *dict)
+		gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
 
@@ -708,9 +724,9 @@ index_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         frame->local = inode_ref (fd->inode);
         stub = fop_fxattrop_stub (frame, index_fxattrop_wrapper,
-                                  fd, flags, dict);
+                                  fd, flags, dict, xdata);
         if (!stub) {
-                INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM, NULL);
+                INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM, NULL, xdata);
                 return 0;
         }
 
@@ -718,17 +734,53 @@ index_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 out:
         STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict);
+                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
         return 0;
 }
 
+uint64_t
+index_entry_count (xlator_t *this, char *subdir)
+{
+	index_priv_t *priv = NULL;
+	char index_dir[PATH_MAX];
+	DIR *dirp = NULL;
+	uint64_t count = 0;
+	struct dirent buf;
+	struct dirent *entry = NULL;
+
+	priv = this->private;
+
+	make_index_dir_path (priv->index_basepath, subdir,
+			     index_dir, sizeof (index_dir));
+
+	dirp = opendir (index_dir);
+	if (!dirp)
+		return 0;
+
+	while (readdir_r (dirp, &buf, &entry) == 0) {
+		if (!entry)
+			break;
+		if (!strcmp (entry->d_name, ".") ||
+		    !strcmp (entry->d_name, ".."))
+			continue;
+                if (!strncmp (entry->d_name, subdir, strlen (subdir)))
+			continue;
+		count++;
+	}
+	closedir (dirp);
+
+	return count;
+}
+
+
 int32_t
 index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, const char *name)
+                        loc_t *loc, const char *name, dict_t *xdata)
 {
         index_priv_t    *priv = NULL;
         dict_t          *xattr = NULL;
         int             ret = 0;
+	uint64_t        count = 0;
 
         priv = this->private;
 
@@ -738,19 +790,31 @@ index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
                 goto done;
         }
 
-        ret = dict_set_static_bin (xattr, (char*)name, priv->xattrop_vgfid,
-                                   sizeof (priv->xattrop_vgfid));
-        if (ret) {
-                ret = -ENOMEM;
-                gf_log (THIS->name, GF_LOG_ERROR, "xattrop index "
-                        "gfid set failed");
-                goto done;
-        }
+	if (strcmp (name, GF_XATTROP_INDEX_GFID) == 0) {
+		ret = dict_set_static_bin (xattr, (char*)name, priv->xattrop_vgfid,
+					   sizeof (priv->xattrop_vgfid));
+		if (ret) {
+			ret = -ENOMEM;
+			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
+				"gfid set failed");
+			goto done;
+		}
+	} else if (strcmp (name, GF_XATTROP_INDEX_COUNT) == 0) {
+		count = index_entry_count (this, XATTROP_SUBDIR);
+
+		ret = dict_set_uint64 (xattr, (char *)name, count);
+		if (ret) {
+			ret = -ENOMEM;
+			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
+				"count set failed");
+			goto done;
+		}
+	}
 done:
         if (ret)
-                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, xattr);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, xattr, xdata);
         else
-                STACK_UNWIND_STRICT (getxattr, frame, 0, 0, xattr);
+                STACK_UNWIND_STRICT (getxattr, frame, 0, 0, xattr, xdata);
 
         if (xattr)
                 dict_unref (xattr);
@@ -820,7 +884,7 @@ done:
 
 int32_t
 index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
-                       fd_t *fd, size_t size, off_t off)
+                       fd_t *fd, size_t size, off_t off, dict_t *xdata)
 {
         index_fd_ctx_t       *fctx           = NULL;
         DIR                  *dir            = NULL;
@@ -855,13 +919,14 @@ index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
         op_errno = errno;
         op_ret = count;
 done:
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
         gf_dirent_free (&entries);
         return 0;
 }
 
 int
-index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc)
+index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                      dict_t *xdata)
 {
         index_priv_t    *priv = NULL;
         int32_t         op_ret = 0;
@@ -905,29 +970,34 @@ index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc)
         postparent.ia_ino = -1;
 done:
         INDEX_STACK_UNWIND (unlink, frame, op_ret, op_errno, &preparent,
-                            &postparent);
+                            &postparent, xdata);
         return 0;
 }
 
 int32_t
 index_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name)
+                loc_t *loc, const char *name, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
 
-        if (!name || strcmp (GF_XATTROP_INDEX_GFID, name))
+        if (!name || (strcmp (GF_XATTROP_INDEX_GFID, name) &&
+		      strcmp (GF_XATTROP_INDEX_COUNT, name)))
                 goto out;
 
-        stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name);
+        stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name,
+                                  xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
         worker_enqueue (this, stub);
         return 0;
 out:
         STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
         return 0;
 }
 
@@ -961,7 +1031,7 @@ normal:
 
 int32_t
 index_readdir (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t off)
+               fd_t *fd, size_t size, off_t off, dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         index_priv_t    *priv = NULL;
@@ -969,21 +1039,23 @@ index_readdir (call_frame_t *frame, xlator_t *this,
         priv = this->private;
         if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid))
                 goto out;
-        stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, off);
+        stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, off,
+                                 xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
         worker_enqueue (this, stub);
         return 0;
 out:
         STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off);
+                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
         return 0;
 }
 
 int
-index_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+index_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
         call_stub_t     *stub = NULL;
         index_priv_t    *priv = NULL;
@@ -992,16 +1064,17 @@ index_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         if (uuid_compare (loc->pargfid, priv->xattrop_vgfid))
                 goto out;
 
-        stub = fop_unlink_stub (frame, index_unlink_wrapper, loc);
+        stub = fop_unlink_stub (frame, index_unlink_wrapper, loc, xflag, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL,
+                                     NULL);
                 return 0;
         }
         worker_enqueue (this, stub);
         return 0;
 out:
         STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
         return 0;
 }
 
@@ -1021,6 +1094,10 @@ init (xlator_t *this)
         int ret = -1;
         index_priv_t *priv = NULL;
         pthread_t thread;
+        pthread_attr_t  w_attr;
+        gf_boolean_t    mutex_inited = _gf_false;
+        gf_boolean_t    cond_inited  = _gf_false;
+        gf_boolean_t    attr_inited  = _gf_false;
 
 	if (!this->children || this->children->next) {
 		gf_log (this->name, GF_LOG_ERROR,
@@ -1043,16 +1120,23 @@ init (xlator_t *this)
                         "pthread_cond_init failed (%d)", ret);
                 goto out;
         }
+        cond_inited = _gf_true;
 
         if ((ret = pthread_mutex_init(&priv->mutex, NULL)) != 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "pthread_mutex_init failed (%d)", ret);
                 goto out;
         }
+        mutex_inited = _gf_true;
 
-        pthread_attr_init (&priv->w_attr);
-        ret = pthread_attr_setstacksize (&priv->w_attr,
-                                         INDEX_THREAD_STACK_SIZE);
+        if ((ret = pthread_attr_init (&w_attr)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_attr_init failed (%d)", ret);
+                goto out;
+        }
+        attr_inited = _gf_true;
+
+        ret = pthread_attr_setstacksize (&w_attr, INDEX_THREAD_STACK_SIZE);
         if (ret == EINVAL) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "Using default thread stack size");
@@ -1063,14 +1147,31 @@ init (xlator_t *this)
         INIT_LIST_HEAD (&priv->callstubs);
 
         this->private = priv;
-        ret = pthread_create (&thread, &priv->w_attr, index_worker, this);
+
+        ret = index_dir_create (this, XATTROP_SUBDIR);
+        if (ret < 0)
+                goto out;
+
+        ret = gf_thread_create (&thread, &w_attr, index_worker, this);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "Failed to create "
                         "worker thread, aborting");
                 goto out;
         }
+
         ret = 0;
 out:
+        if (ret) {
+                if (cond_inited)
+                        pthread_cond_destroy (&priv->cond);
+                if (mutex_inited)
+                        pthread_mutex_destroy (&priv->mutex);
+                if (priv)
+                        GF_FREE (priv);
+                this->private = NULL;
+        }
+        if (attr_inited)
+                pthread_attr_destroy (&w_attr);
         return ret;
 }
 
@@ -1084,6 +1185,8 @@ fini (xlator_t *this)
                 goto out;
         this->private = NULL;
         LOCK_DESTROY (&priv->lock);
+        pthread_cond_destroy (&priv->cond);
+        pthread_mutex_destroy (&priv->mutex);
         GF_FREE (priv);
 out:
         return;
@@ -1155,8 +1258,7 @@ struct xlator_fops fops = {
         .unlink      = index_unlink
 };
 
-struct xlator_dumpops dumpops = {
-};
+struct xlator_dumpops dumpops;
 
 struct xlator_cbks cbks = {
         .forget         = index_forget,
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
index bbf258f30..661dcdbc4 100644
--- a/xlators/features/index/src/index.h
+++ b/xlators/features/index/src/index.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __INDEX_H__
 #define __INDEX_H__
 
@@ -54,7 +44,6 @@ typedef struct index_priv {
         struct list_head callstubs;
         pthread_mutex_t mutex;
         pthread_cond_t  cond;
-        pthread_attr_t  w_attr;
 } index_priv_t;
 
 #define INDEX_STACK_UNWIND(fop, frame, params ...)      \
diff --git a/xlators/features/locks/src/Makefile.am b/xlators/features/locks/src/Makefile.am
index e39676826..0f79731b4 100644
--- a/xlators/features/locks/src/Makefile.am
+++ b/xlators/features/locks/src/Makefile.am
@@ -1,7 +1,7 @@
 xlator_LTLIBRARIES = locks.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-locks_la_LDFLAGS = -module -avoidversion
+locks_la_LDFLAGS = -module -avoid-version
 
 locks_la_SOURCES = common.c posix.c entrylk.c inodelk.c reservelk.c \
 		   clear.c
@@ -9,9 +9,10 @@ locks_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = locks.h common.h locks-mem-types.h clear.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-fno-strict-aliasing -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src $(GF_CFLAGS) -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/locks/src/clear.c b/xlators/features/locks/src/clear.c
index d64a15b09..75593b898 100644
--- a/xlators/features/locks/src/clear.c
+++ b/xlators/features/locks/src/clear.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -141,15 +131,15 @@ clrlk_parse_args (const char* cmd, clrlk_args *args)
         if ((args->type == CLRLK_TYPE_MAX) || (args->kind == CLRLK_KIND_MAX))
                 goto out;
 
-        /*optional args*/
-        tok = strtok_r (NULL, ".", &sptr);
+        /*optional args, neither range nor basename can 'legally' contain
+         * "/" in them*/
+        tok = strtok_r (NULL, "/", &sptr);
         if (tok)
                 args->opts = gf_strdup (tok);
 
         ret = 0;
 out:
-        if (free_ptr)
-                GF_FREE (free_ptr);
+        GF_FREE (free_ptr);
         return ret;
 }
 
@@ -193,8 +183,8 @@ clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
                                               F_SETLKW, &plock->user_flock,
                                               -1, EAGAIN, NULL);
 
-                                STACK_UNWIND (plock->frame, -1, EAGAIN,
-                                              &plock->user_flock);
+                                STACK_UNWIND_STRICT (lk, plock->frame, -1, EAGAIN,
+                                                     &plock->user_flock, NULL);
 
                         } else {
                                 gcount++;
@@ -261,7 +251,7 @@ blkd:
                               &ilock->user_flock, -1, EAGAIN,
                               ilock->volume);
                 STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
-                                     EAGAIN);
+                                     EAGAIN, NULL);
                 //No need to take lock as the locks are only in one list
                 __pl_inodelk_unref (ilock);
         }
@@ -310,11 +300,13 @@ clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
 {
         pl_entry_lock_t         *elock          = NULL;
         pl_entry_lock_t         *tmp            = NULL;
-        struct list_head        removed         = {0};
         int                     bcount          = 0;
         int                     gcount          = 0;
         int                     ret             = -1;
+        struct list_head        removed;
+        struct list_head        released;
 
+        INIT_LIST_HEAD (&released);
         if (args->kind & CLRLK_BLOCKED)
                 goto blkd;
 
@@ -326,21 +318,30 @@ blkd:
         {
                 list_for_each_entry_safe (elock, tmp, &dom->blocked_entrylks,
                                           blocked_locks) {
-                        if (args->opts &&
-                            strncmp (elock->basename, args->opts,
-                                     strlen (elock->basename)))
-                                continue;
+                        if (args->opts) {
+                                if (!elock->basename ||
+                                    strcmp (elock->basename, args->opts))
+                                        continue;
+                        }
 
                         bcount++;
-                        list_del_init (&elock->domain_list);
-                        STACK_UNWIND_STRICT (entrylk, elock->frame, -1,
-                                             EAGAIN);
-                        GF_FREE ((char *) elock->basename);
-                        GF_FREE (elock);
+
+                        list_del_init (&elock->blocked_locks);
+                        list_add_tail (&elock->blocked_locks, &released);
                 }
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
+        list_for_each_entry_safe (elock, tmp, &released, blocked_locks) {
+                list_del_init (&elock->blocked_locks);
+                entrylk_trace_out (this, elock->frame, elock->volume, NULL, NULL,
+                                   elock->basename, ENTRYLK_LOCK, elock->type,
+                                   -1, EAGAIN);
+                STACK_UNWIND_STRICT (entrylk, elock->frame, -1, EAGAIN, NULL);
+
+		__pl_entrylk_unref (elock);
+        }
+
         if (!(args->kind & CLRLK_GRANTED)) {
                 ret = 0;
                 goto out;
@@ -352,24 +353,22 @@ granted:
         {
                 list_for_each_entry_safe (elock, tmp, &dom->entrylk_list,
                                           domain_list) {
-                        if (!elock->basename)
-                                continue;
-
-                        if (args->opts &&
-                            strncmp (elock->basename, args->opts,
-                                     strlen (elock->basename)))
-                                continue;
+                        if (args->opts) {
+                                if (!elock->basename ||
+                                    strcmp (elock->basename, args->opts))
+                                        continue;
+                        }
 
                         gcount++;
                         list_del_init (&elock->domain_list);
                         list_add_tail (&elock->domain_list, &removed);
+
+			__pl_entrylk_unref (elock);
                 }
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
-        list_for_each_entry_safe (elock, tmp, &removed, domain_list) {
-                grant_blocked_entry_locks (this, pl_inode, elock, dom);
-        }
+	grant_blocked_entry_locks (this, pl_inode, dom);
 
         ret = 0;
 out:
@@ -380,8 +379,8 @@ out:
 
 int
 clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
-                          clrlk_args *args, int *blkd, int *granted,
-                          int *op_errno)
+                                clrlk_args *args, int *blkd, int *granted,
+                                int *op_errno)
 {
         pl_dom_list_t   *dom            = NULL;
         int             ret             = -1;
diff --git a/xlators/features/locks/src/clear.h b/xlators/features/locks/src/clear.h
index 95572a971..511f3f74a 100644
--- a/xlators/features/locks/src/clear.h
+++ b/xlators/features/locks/src/clear.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __CLEAR_H__
 #define __CLEAR_H__
 
diff --git a/xlators/features/locks/src/common.c b/xlators/features/locks/src/common.c
index 0bc7baa30..f6c71c1cf 100644
--- a/xlators/features/locks/src/common.c
+++ b/xlators/features/locks/src/common.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -45,8 +35,9 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock);
 static int
 pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
                         posix_lock_t *old_lock);
+
 static pl_dom_list_t *
-allocate_domain (const char *volume)
+__allocate_domain (const char *volume)
 {
         pl_dom_list_t *dom = NULL;
 
@@ -85,24 +76,26 @@ get_domain (pl_inode_t *pl_inode, const char *volume)
 {
         pl_dom_list_t *dom = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, pl_inode, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, volume, out);
-
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                if (strcmp (dom->domain, volume) == 0)
-                        goto found;
+        GF_VALIDATE_OR_GOTO ("posix-locks", pl_inode, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", volume, out);
 
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                        if (strcmp (dom->domain, volume) == 0)
+                                goto unlock;
+                }
 
+                dom = __allocate_domain (volume);
+                if (dom)
+                        list_add (&dom->inode_list, &pl_inode->dom_list);
         }
-
-        dom = allocate_domain (volume);
-        if (dom)
-                list_add (&dom->inode_list, &pl_inode->dom_list);
-found:
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
         if (dom) {
-                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Domain %s found", volume);
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
         } else {
-                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Domain %s not found", volume);
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
         }
 out:
         return dom;
@@ -143,10 +136,10 @@ __pl_inode_is_empty (pl_inode_t *pl_inode)
 void
 pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame)
 {
-        snprintf (str, size, "Pid=%llu, lk-owner=%s, Transport=%p, Frame=%llu",
+        snprintf (str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
                   (unsigned long long) frame->root->pid,
                   lkowner_utoa (&frame->root->lk_owner),
-                  (void *)frame->root->trans,
+                  frame->root->client,
                   (unsigned long long) frame->root->unique);
 }
 
@@ -180,8 +173,7 @@ pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc)
                   uuid_utoa (inode->gfid), fd,
                   ipath ? ipath : "<nul>");
 
-        if (ipath)
-                GF_FREE (ipath);
+        GF_FREE (ipath);
 }
 
 
@@ -435,46 +427,50 @@ pl_inode_get (xlator_t *this, inode_t *inode)
         pl_inode_t *pl_inode = NULL;
         int         ret = 0;
 
-        ret = inode_ctx_get (inode, this,&tmp_pl_inode);
-        if (ret == 0) {
-                pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-                goto out;
-        }
-        pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
-                              gf_locks_mt_pl_inode_t);
-        if (!pl_inode) {
-                goto out;
-        }
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
+                if (ret == 0) {
+                        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+                        goto unlock;
+                }
+                pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
+                                      gf_locks_mt_pl_inode_t);
+                if (!pl_inode) {
+                        goto unlock;
+                }
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "Allocating new pl inode");
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Allocating new pl inode");
 
-        pthread_mutex_init (&pl_inode->mutex, NULL);
+                pthread_mutex_init (&pl_inode->mutex, NULL);
 
-        INIT_LIST_HEAD (&pl_inode->dom_list);
-        INIT_LIST_HEAD (&pl_inode->ext_list);
-        INIT_LIST_HEAD (&pl_inode->rw_list);
-        INIT_LIST_HEAD (&pl_inode->reservelk_list);
-        INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
-        INIT_LIST_HEAD (&pl_inode->blocked_calls);
+                INIT_LIST_HEAD (&pl_inode->dom_list);
+                INIT_LIST_HEAD (&pl_inode->ext_list);
+                INIT_LIST_HEAD (&pl_inode->rw_list);
+                INIT_LIST_HEAD (&pl_inode->reservelk_list);
+                INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
+                INIT_LIST_HEAD (&pl_inode->blocked_calls);
 
-        inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
+                __inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
+        }
+unlock:
+        UNLOCK (&inode->lock);
 
-out:
         return pl_inode;
 }
 
 
 /* Create a new posix_lock_t */
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
                 gf_lkowner_t *owner, fd_t *fd)
 {
         posix_lock_t *lock = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, flock, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, transport, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, fd, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", flock, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", client, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fd, out);
 
         lock = GF_CALLOC (1, sizeof (posix_lock_t),
                           gf_locks_mt_posix_lock_t);
@@ -490,7 +486,7 @@ new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
         else
                 lock->fl_end = flock->l_start + flock->l_len - 1;
 
-        lock->transport  = transport;
+        lock->client     = client;
         lock->fd_num     = fd_to_fdnum (fd);
         lock->fd         = fd;
         lock->client_pid = client_pid;
@@ -570,7 +566,7 @@ same_owner (posix_lock_t *l1, posix_lock_t *l2)
 {
 
         return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->transport  == l2->transport));
+                (l1->client == l2->client));
 
 }
 
@@ -699,7 +695,7 @@ subtract_locks (posix_lock_t *big, posix_lock_t *small)
         }
 
         GF_ASSERT (0);
-        gf_log (POSIX_LOCKS, GF_LOG_ERROR, "Unexpected case in subtract_locks");
+        gf_log ("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
 
 out:
         if (v.locks[0]) {
@@ -719,6 +715,36 @@ done:
         return v;
 }
 
+static posix_lock_t *
+first_conflicting_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
+{
+        posix_lock_t *l = NULL;
+        posix_lock_t *conf = NULL;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (l->blocked)
+                                continue;
+
+                        if (locks_overlap (l, lock)) {
+                                if (same_owner (l, lock))
+                                        continue;
+
+                                if ((l->fl_type == F_WRLCK) ||
+                                    (lock->fl_type == F_WRLCK)) {
+                                        conf = l;
+                                        goto unlock;
+                                }
+                        }
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        return conf;
+}
+
 /*
   Start searching from {begin}, and return the first lock that
   conflicts, NULL if no conflict
@@ -777,6 +803,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
         struct _values v = { .locks = {0, 0, 0} };
 
         list_for_each_entry_safe (conf, t, &pl_inode->ext_list, list) {
+                if (conf->blocked)
+                        continue;
                 if (!locks_overlap (conf, lock))
                         continue;
 
@@ -785,7 +813,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 sum = add_locks (lock, conf);
 
                                 sum->fl_type    = lock->fl_type;
-                                sum->transport  = lock->transport;
+                                sum->client     = lock->client;
                                 sum->fd_num     = lock->fd_num;
                                 sum->client_pid = lock->client_pid;
                                 sum->owner      = lock->owner;
@@ -794,6 +822,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 __destroy_lock (conf);
 
                                 __destroy_lock (lock);
+                                INIT_LIST_HEAD (&sum->list);
+                                posix_lock_to_flock (sum, &sum->user_flock);
                                 __insert_and_merge (pl_inode, sum);
 
                                 return;
@@ -801,7 +831,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 sum = add_locks (lock, conf);
 
                                 sum->fl_type    = conf->fl_type;
-                                sum->transport  = conf->transport;
+                                sum->client     = conf->client;
                                 sum->fd_num     = conf->fd_num;
                                 sum->client_pid = conf->client_pid;
                                 sum->owner      = conf->owner;
@@ -821,6 +851,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                                 continue;
 
                                         INIT_LIST_HEAD (&v.locks[i]->list);
+                                        posix_lock_to_flock (v.locks[i],
+                                                       &v.locks[i]->user_flock);
                                         __insert_and_merge (pl_inode,
                                                             v.locks[i]);
                                 }
@@ -927,7 +959,8 @@ grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode)
                 pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
                               &lock->user_flock, 0, 0, NULL);
 
-                STACK_UNWIND (lock->frame, 0, 0, &lock->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
+                                     &lock->user_flock, NULL);
 
                 GF_FREE (lock);
         }
@@ -956,7 +989,7 @@ pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
         flock.l_len    = old_lock->user_flock.l_len;
 
 
-        unlock_lock = new_posix_lock (&flock, old_lock->transport,
+        unlock_lock = new_posix_lock (&flock, old_lock->client,
                                       old_lock->client_pid, &old_lock->owner,
                                       old_lock->fd);
         GF_VALIDATE_OR_GOTO (this->name, unlock_lock, out);
@@ -972,7 +1005,8 @@ pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
                 pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
                               &lock->user_flock, 0, 0, NULL);
 
-                STACK_UNWIND (lock->frame, 0, 0, &lock->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
+                                     &lock->user_flock, NULL);
 
                 GF_FREE (lock);
         }
@@ -1055,7 +1089,7 @@ pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
         posix_lock_t *conf = NULL;
 
-        conf = first_overlap (pl_inode, lock);
+        conf = first_conflicting_overlap (pl_inode, lock);
 
         if (conf == NULL) {
                 lock->fl_type = F_UNLCK;
@@ -1064,3 +1098,4 @@ pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
 
         return conf;
 }
+
diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h
index fe5ebada9..5ec630ee8 100644
--- a/xlators/features/locks/src/common.h
+++ b/xlators/features/locks/src/common.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __COMMON_H__
 #define __COMMON_H__
 
@@ -24,12 +14,14 @@
 /*dump locks format strings */
 #define RANGE_FMT               "type=%s, whence=%hd, start=%llu, len=%llu"
 #define ENTRY_FMT               "type=%s on basename=%s"
-#define DUMP_GEN_FMT            "pid = %llu, owner=%s, transport=%p, "
+#define DUMP_GEN_FMT            "pid = %llu, owner=%s, client=%p"
 #define GRNTD_AT                "granted at %s"
 #define BLKD_AT                 "blocked at %s"
-#define DUMP_BLKD_FMT           DUMP_GEN_FMT", "BLKD_AT
-#define DUMP_GRNTD_FMT          DUMP_GEN_FMT", "GRNTD_AT
-#define DUMP_BLKD_GRNTD_FMT     DUMP_GEN_FMT", "BLKD_AT", "GRNTD_AT
+#define CONN_ID                 "connection-id=%s"
+#define DUMP_BLKD_FMT           DUMP_GEN_FMT", "CONN_ID", "BLKD_AT
+#define DUMP_GRNTD_FMT          DUMP_GEN_FMT", "CONN_ID", "GRNTD_AT
+#define DUMP_BLKD_GRNTD_FMT     DUMP_GEN_FMT", "CONN_ID", "BLKD_AT", "GRNTD_AT
+
 #define ENTRY_BLKD_FMT          ENTRY_FMT", "DUMP_BLKD_FMT
 #define ENTRY_GRNTD_FMT         ENTRY_FMT", "DUMP_GRNTD_FMT
 #define ENTRY_BLKD_GRNTD_FMT    ENTRY_FMT", "DUMP_BLKD_GRNTD_FMT
@@ -39,8 +31,10 @@
 #define RANGE_BLKD_GRNTD_FMT    RANGE_FMT", "DUMP_BLKD_GRNTD_FMT
 
 #define SET_FLOCK_PID(flock, lock) ((flock)->l_pid = lock->client_pid)
+
+
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
                 gf_lkowner_t *owner, fd_t *fd);
 
 pl_inode_t *
@@ -73,7 +67,8 @@ pl_dom_list_t *
 get_domain (pl_inode_t *pl_inode, const char *volume);
 
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom);
+grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_dom_list_t *dom);
 
 void
 __delete_inode_lock (pl_inode_lock_t *lock);
@@ -83,14 +78,18 @@ __pl_inodelk_unref (pl_inode_lock_t *lock);
 
 void
 grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_entry_lock_t *unlocked, pl_dom_list_t *dom);
+                           pl_dom_list_t *dom);
 
 void pl_update_refkeeper (xlator_t *this, inode_t *inode);
 
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode);
+__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname);
+int32_t
+get_inodelk_count (xlator_t *this, inode_t *inode, char *domname);
 
 int32_t
+__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode);
+int32_t
 get_entrylk_count (xlator_t *this, inode_t *inode);
 
 void pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
@@ -149,4 +148,11 @@ pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
                      posix_lock_t *lock, int can_block);
 int
 pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
+
+uint32_t
+check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename);
+
+void __pl_inodelk_unref (pl_inode_lock_t *lock);
+void __pl_entrylk_unref (pl_entry_lock_t *lock);
+
 #endif /* __COMMON_H__ */
diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index 3e91b9d94..ea6995627 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -33,11 +23,29 @@
 #include "locks.h"
 #include "common.h"
 
+
+void
+__pl_entrylk_unref (pl_entry_lock_t *lock)
+{
+        lock->ref--;
+        if (!lock->ref) {
+		GF_FREE ((char *)lock->basename);
+                GF_FREE (lock->connection_id);
+                GF_FREE (lock);
+        }
+}
+
+
+static void
+__pl_entrylk_ref (pl_entry_lock_t *lock)
+{
+        lock->ref++;
+}
+
+
 static pl_entry_lock_t *
 new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
-                  void *trans, pid_t client_pid, gf_lkowner_t *owner,
-                  const char *volume)
-
+		  const char *domain, call_frame_t *frame, char *conn_id)
 {
         pl_entry_lock_t *newlock = NULL;
 
@@ -49,14 +57,21 @@ new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
 
         newlock->basename   = basename ? gf_strdup (basename) : NULL;
         newlock->type       = type;
-        newlock->trans      = trans;
-        newlock->volume     = volume;
-        newlock->client_pid = client_pid;
-        newlock->owner      = *owner;
+        newlock->client     = frame->root->client;
+        newlock->client_pid = frame->root->pid;
+        newlock->volume     = domain;
+        newlock->owner      = frame->root->lk_owner;
+	newlock->frame      = frame;
+
+        if (conn_id) {
+                newlock->connection_id = gf_strdup (conn_id);
+        }
 
         INIT_LIST_HEAD (&newlock->domain_list);
         INIT_LIST_HEAD (&newlock->blocked_locks);
+	INIT_LIST_HEAD (&newlock->client_list);
 
+	__pl_entrylk_ref (newlock);
 out:
         return newlock;
 }
@@ -87,42 +102,42 @@ __same_entrylk_owner (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
 {
 
         return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->trans  == l2->trans));
+                (l1->client  == l2->client));
 }
 
 
 /**
- * lock_grantable - is this lock grantable?
+ * entrylk_grantable - is this lock grantable?
  * @inode: inode in which to look
  * @basename: name we're trying to lock
  * @type: type of lock
  */
 static pl_entry_lock_t *
-__lock_grantable (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+__entrylk_grantable (pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-        pl_entry_lock_t *lock = NULL;
+        pl_entry_lock_t *tmp = NULL;
 
         if (list_empty (&dom->entrylk_list))
                 return NULL;
 
-        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-                if (names_conflict (lock->basename, basename))
-                        return lock;
+        list_for_each_entry (tmp, &dom->entrylk_list, domain_list) {
+                if (names_conflict (tmp->basename, lock->basename))
+                        return tmp;
         }
 
         return NULL;
 }
 
 static pl_entry_lock_t *
-__blocked_lock_conflict (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+__blocked_entrylk_conflict (pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-        pl_entry_lock_t *lock = NULL;
+        pl_entry_lock_t *tmp = NULL;
 
         if (list_empty (&dom->blocked_entrylks))
                 return NULL;
 
-        list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-                if (names_conflict (lock->basename, basename))
+        list_for_each_entry (tmp, &dom->blocked_entrylks, blocked_locks) {
+                if (names_conflict (tmp->basename, lock->basename))
                         return lock;
         }
 
@@ -303,7 +318,7 @@ __find_most_matching_lock (pl_dom_list_t *dom, const char *basename)
 }
 
 /**
- * __lock_name - lock a name in a directory
+ * __lock_entrylk - lock a name in a directory
  * @inode: inode for the directory in which to lock
  * @basename: name of the entry to lock
  *            if null, lock the entire directory
@@ -314,88 +329,48 @@ __find_most_matching_lock (pl_dom_list_t *dom, const char *basename)
  */
 
 int
-__lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
-             call_frame_t *frame, pl_dom_list_t *dom, xlator_t *this, int nonblock)
+__lock_entrylk (xlator_t *this, pl_inode_t *pinode, pl_entry_lock_t *lock,
+		int nonblock, pl_dom_list_t *dom)
 {
-        pl_entry_lock_t *lock       = NULL;
-        pl_entry_lock_t *conf       = NULL;
-        void            *trans      = NULL;
-        pid_t            client_pid = 0;
-        int              ret        = -EINVAL;
+        pl_entry_lock_t *conf = NULL;
+        int              ret  = -EAGAIN;
 
-        trans = frame->root->trans;
-        client_pid = frame->root->pid;
-
-        lock = new_entrylk_lock (pinode, basename, type, trans, client_pid,
-                                 &frame->root->lk_owner, dom->domain);
-        if (!lock) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        lock->frame   = frame;
-        lock->this    = this;
-        lock->trans   = trans;
-
-        conf = __lock_grantable (dom, basename, type);
+        conf = __entrylk_grantable (dom, lock);
         if (conf) {
                 ret = -EAGAIN;
-                if (nonblock){
-                        if (lock->basename)
-                                GF_FREE ((char *)lock->basename);
-                        GF_FREE (lock);
+                if (nonblock)
                         goto out;
 
-                }
-
                 gettimeofday (&lock->blkd_time, NULL);
                 list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
 
                 gf_log (this->name, GF_LOG_TRACE,
                         "Blocking lock: {pinode=%p, basename=%s}",
-                        pinode, basename);
+                        pinode, lock->basename);
 
                 goto out;
         }
 
-        if ( __blocked_lock_conflict (dom, basename, type) && !(__owner_has_lock (dom, lock))) {
+        if (__blocked_entrylk_conflict (dom, lock) && !(__owner_has_lock (dom, lock))) {
                 ret = -EAGAIN;
-                if (nonblock) {
-                        if (lock->basename)
-                                GF_FREE ((char *) lock->basename);
-                        GF_FREE (lock);
+                if (nonblock)
                         goto out;
 
-                }
-                lock->frame     = frame;
-                lock->this      = this;
-
                 gettimeofday (&lock->blkd_time, NULL);
                 list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
 
-                gf_log (this->name, GF_LOG_TRACE,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Lock is grantable, but blocking to prevent starvation");
                 gf_log (this->name, GF_LOG_TRACE,
                         "Blocking lock: {pinode=%p, basename=%s}",
-                        pinode, basename);
+                        pinode, lock->basename);
 
-                ret = -EAGAIN;
                 goto out;
         }
-        switch (type) {
-
-        case ENTRYLK_WRLCK:
-                gettimeofday (&lock->granted_time, NULL);
-                list_add_tail (&lock->domain_list, &dom->entrylk_list);
-                break;
 
-        default:
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Invalid type for entrylk specified: %d", type);
-                ret = -EINVAL;
-                goto out;
-        }
+        __pl_entrylk_ref (lock);
+        gettimeofday (&lock->granted_time, NULL);
+        list_add (&lock->domain_list, &dom->entrylk_list);
 
         ret = 0;
 out:
@@ -403,37 +378,36 @@ out:
 }
 
 /**
- * __unlock_name - unlock a name in a directory
+ * __unlock_entrylk - unlock a name in a directory
  * @inode: inode for the directory to unlock in
  * @basename: name of the entry to unlock
  *            if null, unlock the entire directory
  */
 
 pl_entry_lock_t *
-__unlock_name (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+__unlock_entrylk (pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-        pl_entry_lock_t *lock = NULL;
+        pl_entry_lock_t *tmp = NULL;
         pl_entry_lock_t *ret_lock = NULL;
 
-        lock = __find_most_matching_lock (dom, basename);
+        tmp = __find_most_matching_lock (dom, lock->basename);
 
-        if (!lock) {
-                gf_log ("locks", GF_LOG_DEBUG,
+        if (!tmp) {
+                gf_log ("locks", GF_LOG_ERROR,
                         "unlock on %s (type=ENTRYLK_WRLCK) attempted but no matching lock found",
-                        basename);
+                        lock->basename);
                 goto out;
         }
 
-        if (names_equal (lock->basename, basename)
-            && lock->type == type) {
+        if (names_equal (tmp->basename, lock->basename)
+            && tmp->type == lock->type) {
+
+		list_del_init (&tmp->domain_list);
+		ret_lock = tmp;
 
-                if (type == ENTRYLK_WRLCK) {
-                        list_del_init (&lock->domain_list);
-                        ret_lock = lock;
-                }
         } else {
-                gf_log ("locks", GF_LOG_DEBUG,
-                        "Unlock for a non-existing lock!");
+                gf_log ("locks", GF_LOG_ERROR,
+                        "Unlock on %s for a non-existing lock!", lock->basename);
                 goto out;
         }
 
@@ -441,6 +415,32 @@ out:
         return ret_lock;
 }
 
+uint32_t
+check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename)
+{
+        uint32_t        entrylk = 0;
+        pl_inode_t      *pinode = 0;
+        pl_dom_list_t   *dom = NULL;
+        pl_entry_lock_t *conf       = NULL;
+
+        pinode = pl_inode_get (this, parent);
+        if (!pinode)
+                goto out;
+        pthread_mutex_lock (&pinode->mutex);
+        {
+                list_for_each_entry (dom, &pinode->dom_list, inode_list) {
+                        conf = __find_most_matching_lock (dom, basename);
+                        if (conf && conf->basename) {
+                                entrylk = 1;
+                                break;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pinode->mutex);
+
+out:
+        return entrylk;
+}
 
 void
 __grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
@@ -455,27 +455,14 @@ __grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
         INIT_LIST_HEAD (&blocked_list);
         list_splice_init (&dom->blocked_entrylks, &blocked_list);
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list,
-                                  blocked_locks) {
+        list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
 
                 list_del_init (&bl->blocked_locks);
 
-
-                gf_log ("locks", GF_LOG_TRACE,
-                        "Trying to unblock: {pinode=%p, basename=%s}",
-                        pl_inode, bl->basename);
-
-                bl_ret = __lock_name (pl_inode, bl->basename, bl->type,
-                                      bl->frame, dom, bl->this, 0);
+                bl_ret = __lock_entrylk (bl->this, pl_inode, bl, 0, dom);
 
                 if (bl_ret == 0) {
                         list_add (&bl->blocked_locks, granted);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "should never happen");
-                        if (bl->basename)
-                                GF_FREE ((char *)bl->basename);
-                        GF_FREE (bl);
                 }
         }
         return;
@@ -484,7 +471,7 @@ __grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
 /* Grants locks if possible which are blocked on a lock */
 void
 grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_entry_lock_t *unlocked, pl_dom_list_t *dom)
+			   pl_dom_list_t *dom)
 {
         struct list_head  granted_list;
         pl_entry_lock_t  *tmp = NULL;
@@ -494,123 +481,56 @@ grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                __grant_blocked_entry_locks (this, pl_inode, dom, &granted_list);
+                __grant_blocked_entry_locks (this, pl_inode, dom,
+                                             &granted_list);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
         list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
-                list_del_init (&lock->blocked_locks);
-
                 entrylk_trace_out (this, lock->frame, NULL, NULL, NULL,
                                    lock->basename, ENTRYLK_LOCK, lock->type,
                                    0, 0);
 
-                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0);
+                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
+		lock->frame = NULL;
+	}
 
-        }
-
-        GF_FREE ((char *)unlocked->basename);
-        GF_FREE (unlocked);
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+		list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
+			list_del_init (&lock->blocked_locks);
+			__pl_entrylk_unref (lock);
+		}
+	}
+        pthread_mutex_unlock (&pl_inode->mutex);
 
         return;
 }
 
-/**
- * release_entry_locks_for_transport: release all entry locks from this
- * transport for this loc_t
- */
-
-static int
-release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
-                                   pl_dom_list_t *dom, void *trans)
-{
-        pl_entry_lock_t  *lock = NULL;
-        pl_entry_lock_t  *tmp = NULL;
-        struct list_head  granted;
-        struct list_head  released;
-
-        INIT_LIST_HEAD (&granted);
-        INIT_LIST_HEAD (&released);
-
-        pthread_mutex_lock (&pinode->mutex);
-        {
-                list_for_each_entry_safe (lock, tmp, &dom->blocked_entrylks,
-                                          blocked_locks) {
-                        if (lock->trans != trans)
-                                continue;
-
-                        list_del_init (&lock->blocked_locks);
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "releasing lock on  held by "
-                                "{transport=%p}",trans);
-
-                        list_add (&lock->blocked_locks, &released);
-
-                }
-
-                list_for_each_entry_safe (lock, tmp, &dom->entrylk_list,
-                                          domain_list) {
-                        if (lock->trans != trans)
-                                continue;
-
-                        list_del_init (&lock->domain_list);
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "releasing lock on  held by "
-                                "{transport=%p}",trans);
-
-                        GF_FREE ((char *)lock->basename);
-                        GF_FREE (lock);
-                }
-
-                __grant_blocked_entry_locks (this, pinode, dom, &granted);
-
-        }
-
-        pthread_mutex_unlock (&pinode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &released, blocked_locks) {
-                list_del_init (&lock->blocked_locks);
-
-                STACK_UNWIND_STRICT (entrylk, lock->frame, -1, EAGAIN);
-
-                if (lock->basename)
-                        GF_FREE ((char *)lock->basename);
-                GF_FREE (lock);
-
-        }
-
-        list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
-                list_del_init (&lock->blocked_locks);
-
-                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0);
-
-                if (lock->basename)
-                        GF_FREE ((char *)lock->basename);
-                GF_FREE (lock);
-        }
-
-        return 0;
-}
 
 /* Common entrylk code called by pl_entrylk and pl_fentrylk */
 int
 pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                    const char *volume, inode_t *inode, const char *basename,
-                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd)
-{
-        int32_t  op_ret   = -1;
-        int32_t  op_errno = 0;
+                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd,
+                   dict_t *xdata)
 
-        void *        transport = NULL;
-
-        pl_inode_t *     pinode   = NULL;
+{
+        int32_t          op_ret   = -1;
+        int32_t          op_errno = 0;
         int              ret      = -1;
-        pl_entry_lock_t *unlocked = NULL;
         char             unwind   = 1;
+        GF_UNUSED int    dict_ret = -1;
+        pl_inode_t      *pinode   = NULL;
+        pl_entry_lock_t *reqlock  = NULL;
+        pl_entry_lock_t *unlocked = NULL;
+        pl_dom_list_t   *dom      = NULL;
+        char            *conn_id  = NULL;
+        pl_ctx_t        *ctx      = NULL;
+	int              nonblock = 0;
 
-        pl_dom_list_t          *dom = NULL;
+        if (xdata)
+                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
 
         pinode = pl_inode_get (this, inode);
         if (!pinode) {
@@ -618,6 +538,15 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+	if (frame->root->client) {
+		ctx = pl_ctx_get (frame->root->client, this);
+		if (!ctx) {
+			op_errno = ENOMEM;
+			gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+			goto unwind;
+		}
+	}
+
         dom = get_domain (pinode, volume);
         if (!dom){
                 op_errno = ENOMEM;
@@ -626,73 +555,69 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
 
         entrylk_trace_in (this, frame, volume, fd, loc, basename, cmd, type);
 
-        transport = frame->root->trans;
-
-        if (frame->root->lk_owner.len == 0) {
-                /*
-                  this is a special case that means release
-                  all locks from this transport
-                */
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Releasing locks for transport %p", transport);
-
-                release_entry_locks_for_transport (this, pinode, dom, transport);
-                op_ret = 0;
-
-                goto out;
+        reqlock = new_entrylk_lock (pinode, basename, type, dom->domain, frame,
+				    conn_id);
+        if (!reqlock) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
         }
 
         switch (cmd) {
-        case ENTRYLK_LOCK:
-                pthread_mutex_lock (&pinode->mutex);
-                {
-                        ret = __lock_name (pinode, basename, type,
-                                           frame, dom, this, 0);
-                }
-                pthread_mutex_unlock (&pinode->mutex);
-
-                op_errno = -ret;
-                if (ret < 0) {
-                        if (ret == -EAGAIN)
-                                unwind = 0;
-                        else
-                                unwind = 1;
-                        goto out;
-                } else {
-                        op_ret = 0;
-                        op_errno = 0;
-                        unwind = 1;
-                        goto out;
-                }
-
-                break;
-
         case ENTRYLK_LOCK_NB:
-                unwind = 1;
+		nonblock = 1;
+		/* fall through */
+	case ENTRYLK_LOCK:
+		if (ctx)
+			pthread_mutex_lock (&ctx->lock);
                 pthread_mutex_lock (&pinode->mutex);
                 {
-                        ret = __lock_name (pinode, basename, type,
-                                           frame, dom, this, 1);
+			reqlock->pinode = pinode;
+
+                        ret = __lock_entrylk (this, pinode, reqlock, nonblock, dom);
+			if (ret == 0) {
+				reqlock->frame = NULL;
+				op_ret = 0;
+			} else {
+				op_errno = -ret;
+			}
+
+			if (ctx && (!ret || !nonblock))
+				list_add (&reqlock->client_list,
+					  &ctx->entrylk_lockers);
+
+			if (ret == -EAGAIN && !nonblock) {
+				/* blocked */
+				unwind = 0;
+			} else {
+				__pl_entrylk_unref (reqlock);
+			}
                 }
                 pthread_mutex_unlock (&pinode->mutex);
-
-                if (ret < 0) {
-                        op_errno = -ret;
-                        goto out;
-                }
-
-                break;
+		if (ctx)
+			pthread_mutex_unlock (&ctx->lock);
+		break;
 
         case ENTRYLK_UNLOCK:
+		if (ctx)
+			pthread_mutex_lock (&ctx->lock);
                 pthread_mutex_lock (&pinode->mutex);
                 {
-                        unlocked = __unlock_name (dom, basename, type);
+                        unlocked = __unlock_entrylk (dom, reqlock);
+			if (unlocked) {
+				list_del_init (&unlocked->client_list);
+				__pl_entrylk_unref (unlocked);
+				op_ret = 0;
+			} else {
+				op_errno = EINVAL;
+			}
+			__pl_entrylk_unref (reqlock);
                 }
                 pthread_mutex_unlock (&pinode->mutex);
+		if (ctx)
+			pthread_mutex_unlock (&ctx->lock);
 
-                if (unlocked)
-                        grant_blocked_entry_locks (this, pinode, unlocked, dom);
+		grant_blocked_entry_locks (this, pinode, dom);
 
                 break;
 
@@ -702,21 +627,19 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                         "a bug report at http://bugs.gluster.com", cmd);
                 goto out;
         }
-
-        op_ret = 0;
 out:
         pl_update_refkeeper (this, inode);
+
         if (unwind) {
                 entrylk_trace_out (this, frame, volume, fd, loc, basename,
                                    cmd, type, op_ret, op_errno);
-
-                STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+unwind:
+                STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, NULL);
         } else {
                 entrylk_trace_block (this, frame, volume, fd, loc, basename,
                                      cmd, type);
         }
 
-
         return 0;
 }
 
@@ -729,10 +652,10 @@ out:
 int
 pl_entrylk (call_frame_t *frame, xlator_t *this,
             const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type)
+            entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-
-        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd, type, loc, NULL);
+        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd,
+                           type, loc, NULL, xdata);
 
         return 0;
 }
@@ -747,16 +670,98 @@ pl_entrylk (call_frame_t *frame, xlator_t *this,
 int
 pl_fentrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type)
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd,
+                           type, NULL, fd, xdata);
+
+        return 0;
+}
+
+
+static void
+pl_entrylk_log_cleanup (pl_entry_lock_t *lock)
 {
+	pl_inode_t *pinode = NULL;
+        char *path = NULL;
+        char *file = NULL;
+
+	pinode = lock->pinode;
 
-        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd, type, NULL, fd);
+	inode_path (pinode->refkeeper, NULL, &path);
+
+	if (path)
+		file = path;
+	else
+		file = uuid_utoa (pinode->refkeeper->gfid);
+
+	gf_log (THIS->name, GF_LOG_WARNING,
+		"releasing lock on %s held by "
+		"{client=%p, pid=%"PRId64" lk-owner=%s}",
+		file, lock->client, (uint64_t) lock->client_pid,
+		lkowner_utoa (&lock->owner));
+	GF_FREE (path);
+}
+
+
+/* Release all entrylks from this client */
+int
+pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
+{
+        pl_entry_lock_t *tmp = NULL;
+        pl_entry_lock_t *l = NULL;
+	pl_dom_list_t *dom = NULL;
+        pl_inode_t *pinode = NULL;
+
+        struct list_head released;
+
+        INIT_LIST_HEAD (&released);
+
+	pthread_mutex_lock (&ctx->lock);
+        {
+                list_for_each_entry_safe (l, tmp, &ctx->entrylk_lockers,
+					  client_list) {
+                        list_del_init (&l->client_list);
+			list_add_tail (&l->client_list, &released);
+
+			pl_entrylk_log_cleanup (l);
+
+			pinode = l->pinode;
+
+			pthread_mutex_lock (&pinode->mutex);
+			{
+				list_del_init (&l->domain_list);
+                        }
+			pthread_mutex_unlock (&pinode->mutex);
+                }
+	}
+        pthread_mutex_unlock (&ctx->lock);
+
+        list_for_each_entry_safe (l, tmp, &released, client_list) {
+                list_del_init (&l->client_list);
+
+		if (l->frame)
+			STACK_UNWIND_STRICT (entrylk, l->frame, -1, EAGAIN,
+					     NULL);
+
+		pinode = l->pinode;
+
+		dom = get_domain (pinode, l->volume);
+
+		grant_blocked_inode_locks (this, pinode, dom);
+
+		pthread_mutex_lock (&pinode->mutex);
+		{
+			__pl_entrylk_unref (l);
+		}
+		pthread_mutex_unlock (&pinode->mutex);
+        }
 
         return 0;
 }
 
 
-static int32_t
+int32_t
 __get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
 {
         int32_t            count = 0;
@@ -765,24 +770,10 @@ __get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
 
         list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
                 list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s  %s on %s state = Active",
-                                dom->domain,
-                                lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
-                                "ENTRYLK_WRLCK", lock->basename);
                         count++;
                 }
 
                 list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s  %s on %s state = Blocked",
-                                dom->domain,
-                                lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
-                                "ENTRYLK_WRLCK", lock->basename);
                         count++;
                 }
 
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index b8b5643e8..e7093e60e 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -45,12 +35,14 @@ __pl_inodelk_ref (pl_inode_lock_t *lock)
         lock->ref++;
 }
 
-inline void
+void
 __pl_inodelk_unref (pl_inode_lock_t *lock)
 {
         lock->ref--;
-        if (!lock->ref)
+        if (!lock->ref) {
+                GF_FREE (lock->connection_id);
                 GF_FREE (lock);
+        }
 }
 
 /* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't conflict */
@@ -132,7 +124,7 @@ static inline int
 same_inodelk_owner (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
         return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->transport  == l2->transport));
+                (l1->client == l2->client));
 }
 
 /* Returns true if the 2 inodelks conflict with each other */
@@ -212,7 +204,7 @@ __lock_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
         int ret = -EINVAL;
 
         conf = __inodelk_grantable (dom, lock);
-        if (conf){
+        if (conf) {
                 ret = -EAGAIN;
                 if (can_block == 0)
                         goto out;
@@ -240,7 +232,7 @@ __lock_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
                 gettimeofday (&lock->blkd_time, NULL);
                 list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
 
-                gf_log (this->name, GF_LOG_TRACE,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Lock is grantable, but blocking to prevent starvation");
                 gf_log (this->name, GF_LOG_TRACE,
                         "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Blocked",
@@ -298,19 +290,25 @@ __inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
 
         conf = find_matching_inodelk (lock, dom);
         if (!conf) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " Matching lock not found for unlock");
+                gf_log (this->name, GF_LOG_ERROR,
+                        " Matching lock not found for unlock %llu-%llu, by %s "
+                        "on %p", (unsigned long long)lock->fl_start,
+                        (unsigned long long)lock->fl_end,
+                        lkowner_utoa (&lock->owner), lock->client);
                 goto out;
         }
         __delete_inode_lock (conf);
         gf_log (this->name, GF_LOG_DEBUG,
-                " Matching lock found for unlock");
+                " Matching lock found for unlock %llu-%llu, by %s on %p",
+                (unsigned long long)lock->fl_start,
+                (unsigned long long)lock->fl_end, lkowner_utoa (&lock->owner),
+                lock->client);
 
 out:
         return conf;
+}
 
 
-}
 static void
 __grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
                              struct list_head *granted, pl_dom_list_t *dom)
@@ -339,7 +337,8 @@ __grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
 
 /* Grant all inodelks blocked on a lock */
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom)
+grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_dom_list_t *dom)
 {
         struct list_head granted;
         pl_inode_lock_t *lock;
@@ -365,7 +364,8 @@ grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *
                 pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
                               &lock->user_flock, 0, 0, lock->volume);
 
-                STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0);
+                STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0, NULL);
+		lock->frame = NULL;
         }
 
         pthread_mutex_lock (&pl_inode->mutex);
@@ -378,109 +378,107 @@ grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *
         pthread_mutex_unlock (&pl_inode->mutex);
 }
 
-/* Release all inodelks from this transport */
-static int
-release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
-                                  inode_t *inode, void *trans)
+
+static void
+pl_inodelk_log_cleanup (pl_inode_lock_t *lock)
 {
-        pl_inode_lock_t *tmp = NULL;
-        pl_inode_lock_t *l = NULL;
+	pl_inode_t *pl_inode = NULL;
+        char *path = NULL;
+        char *file = NULL;
 
-        pl_inode_t * pinode = NULL;
+	pl_inode = lock->pl_inode;
 
-        struct list_head released;
+	inode_path (pl_inode->refkeeper, NULL, &path);
 
-        char *path = NULL;
-        char *file = NULL;
+	if (path)
+		file = path;
+	else
+		file = uuid_utoa (pl_inode->refkeeper->gfid);
 
-        INIT_LIST_HEAD (&released);
+	gf_log (THIS->name, GF_LOG_WARNING,
+		"releasing lock on %s held by "
+		"{client=%p, pid=%"PRId64" lk-owner=%s}",
+		file, lock->client, (uint64_t) lock->client_pid,
+		lkowner_utoa (&lock->owner));
+	GF_FREE (path);
+}
 
-        pinode = pl_inode_get (this, inode);
 
-        pthread_mutex_lock (&pinode->mutex);
-        {
+/* Release all entrylks from this client */
+int
+pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
+{
+        pl_inode_lock_t *tmp = NULL;
+        pl_inode_lock_t *l = NULL;
+	pl_dom_list_t *dom = NULL;
+        pl_inode_t *pl_inode = NULL;
 
-                list_for_each_entry_safe (l, tmp, &dom->blocked_inodelks, blocked_locks) {
-                        if (l->transport != trans)
-                                continue;
+        struct list_head released;
 
-                        list_del_init (&l->blocked_locks);
+        INIT_LIST_HEAD (&released);
 
-                        inode_path (inode, NULL, &path);
-                        if (path)
-                                file = path;
-                        else
-                                file = uuid_utoa (inode->gfid);
+	pthread_mutex_lock (&ctx->lock);
+        {
+                list_for_each_entry_safe (l, tmp, &ctx->inodelk_lockers,
+					  client_list) {
+                        list_del_init (&l->client_list);
+			list_add_tail (&l->client_list, &released);
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "releasing blocking lock on %s held by "
-                                "{transport=%p, pid=%"PRId64" lk-owner=%s}",
-                                file, trans, (uint64_t) l->client_pid,
-                                lkowner_utoa (&l->owner));
+			pl_inodelk_log_cleanup (l);
 
-                        list_add (&l->blocked_locks, &released);
-                        if (path) {
-                                GF_FREE (path);
-                                path = NULL;
+			pl_inode = l->pl_inode;
+
+			pthread_mutex_lock (&pl_inode->mutex);
+			{
+				__delete_inode_lock (l);
                         }
+			pthread_mutex_unlock (&pl_inode->mutex);
                 }
+	}
+        pthread_mutex_unlock (&ctx->lock);
 
-                list_for_each_entry_safe (l, tmp, &dom->inodelk_list, list) {
-                        if (l->transport != trans)
-                                continue;
-
-                        inode_path (inode, NULL, &path);
-                        if (path)
-                                file = path;
-                        else
-                                file = uuid_utoa (inode->gfid);
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "releasing granted lock on %s held by "
-                                "{transport=%p, pid=%"PRId64" lk-owner=%s}",
-                                file, trans, (uint64_t) l->client_pid,
-                                lkowner_utoa (&l->owner));
-
-                        if (path) {
-                                GF_FREE (path);
-                                path = NULL;
-                        }
+        list_for_each_entry_safe (l, tmp, &released, client_list) {
+                list_del_init (&l->client_list);
 
-                        __delete_inode_lock (l);
-                        __pl_inodelk_unref (l);
-                }
-        }
-        if (path)
-                GF_FREE (path);
+		if (l->frame)
+			STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN,
+					     NULL);
+
+		pl_inode = l->pl_inode;
 
-        pthread_mutex_unlock (&pinode->mutex);
+		dom = get_domain (pl_inode, l->volume);
 
-        list_for_each_entry_safe (l, tmp, &released, blocked_locks) {
-                list_del_init (&l->blocked_locks);
+		grant_blocked_inode_locks (this, pl_inode, dom);
 
-                STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN);
-                //No need to take lock as the locks are only in one list
-                __pl_inodelk_unref (l);
+		pthread_mutex_lock (&pl_inode->mutex);
+		{
+			__pl_inodelk_unref (l);
+		}
+		pthread_mutex_unlock (&pl_inode->mutex);
         }
 
-        grant_blocked_inode_locks (this, pinode, dom);
         return 0;
 }
 
 
 static int
-pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
-                int can_block,  pl_dom_list_t *dom)
+pl_inode_setlk (xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode,
+		pl_inode_lock_t *lock, int can_block, pl_dom_list_t *dom)
 {
         int ret = -EINVAL;
         pl_inode_lock_t *retlock = NULL;
         gf_boolean_t    unref = _gf_true;
 
+	lock->pl_inode = pl_inode;
+
+	if (ctx)
+		pthread_mutex_lock (&ctx->lock);
         pthread_mutex_lock (&pl_inode->mutex);
         {
                 if (lock->fl_type != F_UNLCK) {
                         ret = __lock_inodelk (this, pl_inode, lock, can_block, dom);
                         if (ret == 0) {
+				lock->frame = NULL;
                                 gf_log (this->name, GF_LOG_TRACE,
                                         "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
                                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
@@ -499,6 +497,10 @@ pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
                                 if (can_block)
                                         unref = _gf_false;
                         }
+
+			if (ctx && (!ret || can_block))
+				list_add_tail (&lock->client_list,
+					       &ctx->inodelk_lockers);
                 } else {
                         retlock = __inode_unlock_lock (this, lock, dom);
                         if (!retlock) {
@@ -507,23 +509,29 @@ pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
                                 ret = -EINVAL;
                                 goto out;
                         }
-                        __pl_inodelk_unref (retlock);
+			list_del_init (&retlock->client_list);
+			__pl_inodelk_unref (retlock);
 
                         ret = 0;
                 }
-        }
 out:
-        if (unref)
-                __pl_inodelk_unref (lock);
+		if (unref)
+			__pl_inodelk_unref (lock);
+        }
         pthread_mutex_unlock (&pl_inode->mutex);
+	if (ctx)
+		pthread_mutex_unlock (&ctx->lock);
+
         grant_blocked_inode_locks (this, pl_inode, dom);
+
         return ret;
 }
 
 /* Create a new inode_lock_t */
 pl_inode_lock_t *
-new_inode_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
-                gf_lkowner_t *owner, const char *volume)
+new_inode_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
+                call_frame_t *frame, xlator_t *this, const char *volume,
+                char *conn_id)
 
 {
         pl_inode_lock_t *lock = NULL;
@@ -542,33 +550,77 @@ new_inode_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
         else
                 lock->fl_end = flock->l_start + flock->l_len - 1;
 
-        lock->transport  = transport;
+        lock->client     = client;
         lock->client_pid = client_pid;
         lock->volume     = volume;
-        lock->owner      = *owner;
+        lock->owner      = frame->root->lk_owner;
+        lock->frame      = frame;
+        lock->this       = this;
+
+        if (conn_id) {
+                lock->connection_id = gf_strdup (conn_id);
+        }
 
         INIT_LIST_HEAD (&lock->list);
         INIT_LIST_HEAD (&lock->blocked_locks);
+	INIT_LIST_HEAD (&lock->client_list);
         __pl_inodelk_ref (lock);
 
         return lock;
 }
 
+int32_t
+_pl_convert_volume (const char *volume, char **res)
+{
+        char    *mdata_vol = NULL;
+        int     ret = 0;
+
+        mdata_vol = strrchr (volume, ':');
+        //if the volume already ends with :metadata don't bother
+        if (mdata_vol && (strcmp (mdata_vol, ":metadata") == 0))
+                return 0;
+
+        ret = gf_asprintf (res, "%s:metadata", volume);
+        if (ret <= 0)
+                return ENOMEM;
+        return 0;
+}
+
+int32_t
+_pl_convert_volume_for_special_range (struct gf_flock *flock,
+                                      const char *volume, char **res)
+{
+        int32_t     ret = 0;
+
+        if ((flock->l_start == LLONG_MAX -1) &&
+            (flock->l_len == 0)) {
+                ret = _pl_convert_volume (volume, res);
+        }
+
+        return ret;
+}
+
 /* Common inodelk code called from pl_inodelk and pl_finodelk */
 int
 pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                    const char *volume, inode_t *inode, int32_t cmd,
-                   struct gf_flock *flock, loc_t *loc, fd_t *fd)
+                   struct gf_flock *flock, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         int32_t           op_ret     = -1;
         int32_t           op_errno   = 0;
         int               ret        = -1;
+        GF_UNUSED int     dict_ret   = -1;
         int               can_block  = 0;
-        pid_t             client_pid = -1;
-        void *            transport  = NULL;
         pl_inode_t *      pinode     = NULL;
         pl_inode_lock_t * reqlock    = NULL;
         pl_dom_list_t *   dom        = NULL;
+        char             *res        = NULL;
+        char             *res1       = NULL;
+        char             *conn_id    = NULL;
+        pl_ctx_t         *ctx        = NULL;
+
+        if (xdata)
+                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (inode, unwind);
@@ -579,10 +631,22 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
+        op_errno = _pl_convert_volume_for_special_range (flock, volume, &res);
+        if (op_errno)
+                goto unwind;
+        if (res)
+                volume = res;
+
         pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
 
-        transport  = frame->root->trans;
-        client_pid = frame->root->pid;
+	if (frame->root->client) {
+		ctx = pl_ctx_get (frame->root->client, this);
+		if (!ctx) {
+			op_errno = ENOMEM;
+			gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+			goto unwind;
+		}
+	}
 
         pinode = pl_inode_get (this, inode);
         if (!pinode) {
@@ -591,23 +655,13 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
         }
 
         dom = get_domain (pinode, volume);
-
-        if (frame->root->lk_owner.len == 0) {
-                /*
-                  special case: this means release all locks
-                  from this transport
-                */
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Releasing all locks from transport %p", transport);
-
-                release_inode_locks_of_transport (this, dom, inode, transport);
-
-                op_ret = 0;
+        if (!dom) {
+                op_errno = ENOMEM;
                 goto unwind;
         }
 
-        reqlock = new_inode_lock (flock, transport, client_pid,
-                                  &frame->root->lk_owner, volume);
+        reqlock = new_inode_lock (flock, frame->root->client, frame->root->pid,
+                                  frame, this, volume, conn_id);
 
         if (!reqlock) {
                 op_ret = -1;
@@ -615,21 +669,17 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        reqlock->frame = frame;
-        reqlock->this  = this;
 
         switch (cmd) {
         case F_SETLKW:
                 can_block = 1;
-                reqlock->frame = frame;
-                reqlock->this  = this;
 
                 /* fall through */
 
         case F_SETLK:
                 memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
-                ret = pl_inode_setlk (this, pinode, reqlock,
-                                      can_block, dom);
+                ret = pl_inode_setlk (this, ctx, pinode, reqlock, can_block,
+				      dom);
 
                 if (ret < 0) {
                         if ((can_block) && (F_UNLCK != flock->l_type)) {
@@ -660,80 +710,80 @@ unwind:
                 pl_trace_out (this, frame, fd, loc, cmd, flock, op_ret, op_errno, volume);
         }
 
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, NULL);
 out:
+        GF_FREE (res);
+        GF_FREE (res1);
         return 0;
 }
 
 int
 pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
+            dict_t *xdata)
 {
-
-        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock, loc, NULL);
+        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock,
+                           loc, NULL, xdata);
 
         return 0;
 }
 
 int
 pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+             dict_t *xdata)
 {
-
-        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock, NULL, fd);
+        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock,
+                           NULL, fd, xdata);
 
         return 0;
 
 }
 
+static inline int32_t
+__get_inodelk_dom_count (pl_dom_list_t *dom)
+{
+        pl_inode_lock_t     *lock   = NULL;
+        int32_t             count   = 0;
 
-static int32_t
-__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode)
+        list_for_each_entry (lock, &dom->inodelk_list, list) {
+                count++;
+        }
+        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                count++;
+        }
+        return count;
+}
+
+/* Returns the no. of locks (blocked/granted) held on a given domain name
+ * If @domname is NULL, returns the no. of locks in all the domains present.
+ * If @domname is non-NULL and non-existent, returns 0 */
+int32_t
+__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname)
 {
         int32_t            count  = 0;
-        pl_inode_lock_t   *lock   = NULL;
         pl_dom_list_t     *dom    = NULL;
 
         list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                list_for_each_entry (lock, &dom->inodelk_list, list) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s %s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" "
-                                "state = Active",
-                                dom->domain,
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-
-                        count++;
-                }
-
-                list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                if (domname) {
+                        if (strcmp (domname, dom->domain) == 0) {
+                                count = __get_inodelk_dom_count (dom);
+                                goto out;
+                        }
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s %s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" "
-                                "state = Blocked",
-                                dom->domain,
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
+                } else {
+                    /* Counting locks from all domains */
+                        count += __get_inodelk_dom_count (dom);
 
-                        count++;
                 }
-
         }
 
+out:
         return count;
 }
 
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode)
+get_inodelk_count (xlator_t *this, inode_t *inode, char *domname)
 {
         pl_inode_t   *pl_inode = NULL;
         uint64_t      tmp_pl_inode = 0;
@@ -749,7 +799,7 @@ get_inodelk_count (xlator_t *this, inode_t *inode)
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                count = __get_inodelk_count (this, pl_inode);
+                count = __get_inodelk_count (this, pl_inode, domname);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
diff --git a/xlators/features/locks/src/locks-mem-types.h b/xlators/features/locks/src/locks-mem-types.h
index 5b29cbc72..08aeb0a79 100644
--- a/xlators/features/locks/src/locks-mem-types.h
+++ b/xlators/features/locks/src/locks-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __LOCKS_MEM_TYPES_H__
 #define __LOCKS_MEM_TYPES_H__
 
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index 6df1235a2..8c2a6f867 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __POSIX_LOCKS_H__
 #define __POSIX_LOCKS_H__
 
@@ -29,10 +19,10 @@
 #include "stack.h"
 #include "call-stub.h"
 #include "locks-mem-types.h"
+#include "client_t.h"
 
 #include "lkowner.h"
 
-#define POSIX_LOCKS "posix-locks"
 struct __pl_fd;
 
 struct __posix_lock {
@@ -43,7 +33,7 @@ struct __posix_lock {
         off_t              fl_end;
 
         short              blocked;    /* waiting to acquire */
-        struct gf_flock       user_flock; /* the flock supplied by the user */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
         xlator_t          *this;       /* required for blocked locks */
         unsigned long      fd_num;
 
@@ -56,7 +46,7 @@ struct __posix_lock {
         /* These two together serve to uniquely identify each process
            across nodes */
 
-        void              *transport;     /* to identify client node */
+        void              *client;     /* to identify client node */
         gf_lkowner_t       owner;
         pid_t              client_pid;    /* pid of client process */
 };
@@ -73,9 +63,9 @@ struct __pl_inode_lock {
 
         const char        *volume;
 
-        struct gf_flock       user_flock; /* the flock supplied by the user */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
         xlator_t          *this;       /* required for blocked locks */
-        fd_t              *fd;
+	struct __pl_inode *pl_inode;
 
         call_frame_t      *frame;
 
@@ -85,9 +75,13 @@ struct __pl_inode_lock {
         /* These two together serve to uniquely identify each process
            across nodes */
 
-        void              *transport;     /* to identify client node */
+        void              *client;     /* to identify client node */
         gf_lkowner_t       owner;
         pid_t              client_pid;    /* pid of client process */
+
+        char              *connection_id; /* stores the client connection id */
+
+	struct list_head   client_list; /* list of all locks from a client */
 };
 typedef struct __pl_inode_lock pl_inode_lock_t;
 
@@ -111,9 +105,11 @@ typedef struct __pl_dom_list_t pl_dom_list_t;
 struct __entry_lock {
         struct list_head  domain_list;    /* list_head back to pl_dom_list_t */
         struct list_head  blocked_locks; /* list_head back to blocked_entrylks */
+	int ref;
 
         call_frame_t     *frame;
         xlator_t         *this;
+	struct __pl_inode *pinode;
 
         const char       *volume;
 
@@ -123,9 +119,13 @@ struct __entry_lock {
         struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
         struct timeval     granted_time; /*time at which lock was queued into active list*/
 
-        void             *trans;
+        void             *client;
         gf_lkowner_t      owner;
-        pid_t             client_pid;    /* pid of client process */
+	pid_t             client_pid;    /* pid of client process */
+
+        char             *connection_id; /* stores the client connection id */
+
+	struct list_head   client_list; /* list of all locks from a client */
 };
 typedef struct __entry_lock pl_entry_lock_t;
 
@@ -150,31 +150,55 @@ struct __pl_inode {
 typedef struct __pl_inode pl_inode_t;
 
 
-struct __pl_fd {
-        gf_boolean_t nonblocking;       /* whether O_NONBLOCK has been set */
-};
-typedef struct __pl_fd pl_fd_t;
-
-
 typedef struct {
         gf_boolean_t    mandatory;      /* if mandatory locking is enabled */
         gf_boolean_t    trace;          /* trace lock requests in and out */
+        char           *brickname;
 } posix_locks_private_t;
 
+
 typedef struct {
         gf_boolean_t   entrylk_count_req;
         gf_boolean_t   inodelk_count_req;
+        gf_boolean_t   inodelk_dom_count_req;
         gf_boolean_t   posixlk_count_req;
+        gf_boolean_t   parent_entrylk_req;
 
         /* used by {f,}truncate */
         loc_t  loc;
         fd_t  *fd;
         off_t  offset;
+        dict_t *xdata;
         enum {TRUNCATE, FTRUNCATE} op;
 } pl_local_t;
 
+
 typedef struct {
         struct list_head locks_list;
 } pl_fdctx_t;
 
+
+struct _locker {
+        struct list_head  lockers;
+        char             *volume;
+        inode_t          *inode;
+        gf_lkowner_t      owner;
+};
+
+typedef struct _locks_ctx {
+        pthread_mutex_t      lock;
+        struct list_head     inodelk_lockers;
+        struct list_head     entrylk_lockers;
+} pl_ctx_t;
+
+
+pl_ctx_t *
+pl_ctx_get (client_t *client, xlator_t *xlator);
+
+int
+pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx);
+
+int
+pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx);
+
 #endif /* __POSIX_LOCKS_H__ */
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index 39991299c..2db327687 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -38,6 +28,8 @@
 #include "common.h"
 #include "statedump.h"
 #include "clear.h"
+#include "defaults.h"
+#include "syncop.h"
 
 #ifndef LLONG_MAX
 #define LLONG_MAX LONG_LONG_MAX /* compat with old gcc */
@@ -48,6 +40,9 @@
 
 void do_blocked_rw (pl_inode_t *);
 static int __rw_allowable (pl_inode_t *, posix_lock_t *, glusterfs_fop_t);
+static int format_brickname(char *);
+int pl_lockinfo_get_brickname (xlator_t *, inode_t *, int32_t *);
+static int fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **);
 
 static pl_fdctx_t *
 pl_new_fdctx ()
@@ -56,7 +51,7 @@ pl_new_fdctx ()
 
         fdctx = GF_CALLOC (1, sizeof (*fdctx),
                            gf_locks_mt_pl_fdctx_t);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, fdctx, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fdctx, out);
 
         INIT_LIST_HEAD (&fdctx->locks_list);
 
@@ -71,7 +66,7 @@ pl_check_n_create_fdctx (xlator_t *this, fd_t *fd)
         uint64_t    tmp   = 0;
         pl_fdctx_t *fdctx = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, this, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", this, out);
         GF_VALIDATE_OR_GOTO (this->name, fd, out);
 
         LOCK (&fd->lock);
@@ -102,7 +97,7 @@ out:
 int
 pl_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
+                 struct iatt *postbuf, dict_t *xdata)
 {
         pl_local_t *local = NULL;
 
@@ -111,15 +106,20 @@ pl_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->op == TRUNCATE)
                 loc_wipe (&local->loc);
 
+        if (local->xdata)
+                dict_unref (local->xdata);
+        if (local->fd)
+                fd_unref (local->fd);
+
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 static int
 truncate_allowed (pl_inode_t *pl_inode,
-                  void *transport, pid_t client_pid,
+                  client_t *client, pid_t client_pid,
                   gf_lkowner_t *owner, off_t offset)
 {
         posix_lock_t *l = NULL;
@@ -128,7 +128,7 @@ truncate_allowed (pl_inode_t *pl_inode,
 
         region.fl_start   = offset;
         region.fl_end     = LLONG_MAX;
-        region.transport  = transport;
+        region.client     = client;
         region.client_pid = client_pid;
         region.owner      = *owner;
 
@@ -139,7 +139,7 @@ truncate_allowed (pl_inode_t *pl_inode,
                             && locks_overlap (&region, l)
                             && !same_owner (&region, l)) {
                                 ret = 0;
-                                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Truncate "
+                                gf_log ("posix-locks", GF_LOG_TRACE, "Truncate "
                                         "allowed");
                                 break;
                         }
@@ -153,7 +153,8 @@ truncate_allowed (pl_inode_t *pl_inode,
 
 static int
 truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
         pl_local_t            *local = NULL;
@@ -185,7 +186,7 @@ truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (priv->mandatory
             && pl_inode->mandatory
-            && !truncate_allowed (pl_inode, frame->root->trans,
+            && !truncate_allowed (pl_inode, frame->root->client,
                                   frame->root->pid, &frame->root->lk_owner,
                                   local->offset)) {
                 op_ret   = -1;
@@ -197,12 +198,12 @@ truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         case TRUNCATE:
                 STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->truncate,
-                            &local->loc, local->offset);
+                            &local->loc, local->offset, local->xdata);
                 break;
         case FTRUNCATE:
                 STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->ftruncate,
-                            local->fd, local->offset);
+                            local->fd, local->offset, local->xdata);
                 break;
         }
 
@@ -213,15 +214,19 @@ unwind:
                 "error: %s", op_ret, strerror (op_errno));
         if (local->op == TRUNCATE)
                 loc_wipe (&local->loc);
+        if (local->xdata)
+                dict_unref (local->xdata);
+        if (local->fd)
+                fd_unref (local->fd);
 
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, NULL, xdata);
         return 0;
 }
 
 
 int
 pl_truncate (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, off_t offset)
+             loc_t *loc, off_t offset, dict_t *xdata)
 {
         pl_local_t *local = NULL;
 
@@ -231,18 +236,20 @@ pl_truncate (call_frame_t *frame, xlator_t *this,
         local->op         = TRUNCATE;
         local->offset     = offset;
         loc_copy (&local->loc, loc);
+        if (xdata)
+                local->xdata = dict_ref (xdata);
 
         frame->local = local;
 
         STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc);
+                    FIRST_CHILD (this)->fops->stat, loc, NULL);
 
         return 0;
 
 unwind:
         gf_log (this->name, GF_LOG_ERROR, "truncate for %s failed with ret: %d, "
                 "error: %s", loc->path, -1, strerror (ENOMEM));
-        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -250,7 +257,7 @@ unwind:
 
 int
 pl_ftruncate (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, off_t offset)
+              fd_t *fd, off_t offset, dict_t *xdata)
 {
         pl_local_t *local = NULL;
 
@@ -259,18 +266,20 @@ pl_ftruncate (call_frame_t *frame, xlator_t *this,
 
         local->op         = FTRUNCATE;
         local->offset     = offset;
-        local->fd         = fd;
+        local->fd         = fd_ref (fd);
+        if (xdata)
+                local->xdata = dict_ref (xdata);
 
         frame->local = local;
 
         STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
         return 0;
 
 unwind:
         gf_log (this->name, GF_LOG_ERROR, "ftruncate failed with ret: %d, "
                 "error: %s", -1, strerror (ENOMEM));
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -325,7 +334,8 @@ delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 
        list_for_each_entry_safe (l, tmp, &blocked_list, list) {
                list_del_init(&l->list);
-               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock);
+               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock,
+                                    NULL);
                __destroy_lock (l);
        }
 
@@ -337,7 +347,7 @@ delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 
 static void
 __delete_locks_of_owner (pl_inode_t *pl_inode,
-                         void *transport, gf_lkowner_t *owner)
+                         client_t *client, gf_lkowner_t *owner)
 {
         posix_lock_t *tmp = NULL;
         posix_lock_t *l = NULL;
@@ -345,7 +355,9 @@ __delete_locks_of_owner (pl_inode_t *pl_inode,
         /* TODO: what if it is a blocked lock with pending l->frame */
 
         list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if ((l->transport == transport) &&
+                if (l->blocked)
+                        continue;
+                if ((l->client == client) &&
                     is_same_lkowner (&l->owner, owner)) {
                         gf_log ("posix-locks", GF_LOG_TRACE,
                                 " Flushing lock"
@@ -368,18 +380,18 @@ __delete_locks_of_owner (pl_inode_t *pl_inode,
 
 int32_t
 pl_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict)
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 
 }
 
 int32_t
 pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             const char *name)
+             const char *name, dict_t *xdata)
 {
-        int                     op_errno        = EINVAL;
+        int32_t                 op_errno        = EINVAL;
         int                     op_ret          = -1;
         int32_t                 bcount          = 0;
         int32_t                 gcount          = 0;
@@ -387,7 +399,8 @@ pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         char                    *lk_summary     = NULL;
         pl_inode_t              *pl_inode       = NULL;
         dict_t                  *dict           = NULL;
-        clrlk_args              args           = {0,};
+        clrlk_args              args            = {0,};
+        char                    *brickname      = NULL;
 
         if (!name)
                 goto usual;
@@ -434,33 +447,51 @@ pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                         goto out;
         }
 
+        op_ret = fetch_pathinfo (this, loc->inode, &op_errno, &brickname);
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Couldn't get brickname");
+        } else {
+                op_ret = format_brickname(brickname);
+                if (op_ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Couldn't format brickname");
+                        GF_FREE(brickname);
+                        brickname = NULL;
+                }
+        }
+
         if (!gcount && !bcount) {
                 if (gf_asprintf (&lk_summary, "No locks cleared.") == -1) {
+                        op_ret = -1;
                         op_errno = ENOMEM;
                         goto out;
                 }
         } else if (gf_asprintf (&lk_summary, "%s: %s blocked locks=%d "
-                                "granted locks=%d", this->name,
+                                "granted locks=%d",
+                                (brickname == NULL)? this->name : brickname,
                                 (args.type == CLRLK_INODE)? "inode":
                                 (args.type == CLRLK_ENTRY)? "entry":
                                 (args.type == CLRLK_POSIX)? "posix": " ",
                                 bcount, gcount) == -1) {
+                op_ret = -1;
                 op_errno = ENOMEM;
                 goto out;
         }
 
         strncpy (key, name, strlen (name));
         if (dict_set_dynstr (dict, key, lk_summary)) {
+                op_ret = -1;
                 op_errno = ENOMEM;
                 goto out;
         }
 
         op_ret = 0;
 out:
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        GF_FREE(brickname);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
 
-        if (args.opts)
-                GF_FREE (args.opts);
+        GF_FREE (args.opts);
         if (op_ret && lk_summary)
                 GF_FREE (lk_summary);
         if (dict)
@@ -469,17 +500,416 @@ out:
 
 usual:
         STACK_WIND (frame, pl_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+}
+
+static int
+format_brickname(char *brickname)
+{
+       int   ret             = -1;
+       char *hostname        = NULL;
+       char *volume          = NULL;
+       char *saveptr         = NULL;
+
+       if (!brickname)
+               goto out;
+
+       strtok_r(brickname, ":", &saveptr);
+       hostname = gf_strdup(strtok_r(NULL, ":", &saveptr));
+       if (hostname == NULL)
+               goto out;
+       volume = gf_strdup(strtok_r(NULL, ".", &saveptr));
+       if (volume == NULL)
+               goto out;
+
+       sprintf(brickname, "%s:%s", hostname, volume);
+
+       ret = 0;
+out:
+       GF_FREE(hostname);
+       GF_FREE(volume);
+       return ret;
+}
+
+static int
+fetch_pathinfo (xlator_t *this, inode_t *inode, int32_t *op_errno,
+                char **brickname)
+{
+        int                    ret       = -1;
+        loc_t                  loc       = {0, };
+        dict_t                *dict      = NULL;
+
+        if (!brickname)
+                goto out;
+
+        if (!op_errno)
+                goto out;
+
+        uuid_copy (loc.gfid, inode->gfid);
+        loc.inode = inode_ref (inode);
+
+        ret = syncop_getxattr (FIRST_CHILD(this), &loc, &dict,
+                               GF_XATTR_PATHINFO_KEY);
+        if (ret < 0) {
+                *op_errno = -ret;
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, brickname);
+        if (ret)
+                goto out;
+
+        *brickname = gf_strdup(*brickname);
+        if (*brickname == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (dict != NULL) {
+                dict_unref (dict);
+        }
+        loc_wipe(&loc);
+
+        return ret;
+}
+
+
+int
+pl_lockinfo_get_brickname (xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+        int                    ret       = -1;
+        posix_locks_private_t *priv      = NULL;
+        char                  *brickname = NULL;
+        char                  *end       = NULL;
+        char                  *tmp       = NULL;
+
+        priv = this->private;
+
+        ret = fetch_pathinfo (this, inode, op_errno, &brickname);
+        if (ret)
+                goto out;
+
+        end = strrchr (brickname, ':');
+        if (!end) {
+                GF_FREE(brickname);
+                ret = -1;
+                goto out;
+        }
+
+        tmp = brickname;
+        brickname = gf_strndup (brickname, (end - brickname));
+        if (brickname == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        priv->brickname = brickname;
+        ret = 0;
+out:
+        GF_FREE(tmp);
+        return ret;
+}
+
+char *
+pl_lockinfo_key (xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+        posix_locks_private_t *priv = NULL;
+        char                  *key  = NULL;
+        int                    ret = 0;
+
+        priv = this->private;
+
+        if (priv->brickname == NULL) {
+                ret = pl_lockinfo_get_brickname (this, inode, op_errno);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot get brickname");
+                        goto out;
+                }
+        }
+
+        key = priv->brickname;
+out:
+        return key;
+}
+
+int32_t
+pl_fgetxattr_handle_lockinfo (xlator_t *this, fd_t *fd,
+                              dict_t *dict, int32_t *op_errno)
+{
+        pl_inode_t    *pl_inode = NULL;
+        char          *key      = NULL, *buf = NULL;
+        int32_t        op_ret   = 0;
+        unsigned long  fdnum    = 0, len = 0;
+        dict_t        *tmp      = NULL;
+
+        pl_inode = pl_inode_get (this, fd->inode);
+
+        if (!pl_inode) {
+                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
+                *op_errno = EBADFD;
+                op_ret = -1;
+                goto out;
+        }
+
+        if (!pl_locks_by_fd (pl_inode, fd)) {
+                op_ret = 0;
+                goto out;
+        }
+
+        fdnum = fd_to_fdnum (fd);
+
+        key = pl_lockinfo_key (this, fd->inode, op_errno);
+        if (key == NULL) {
+                op_ret = -1;
+                goto out;
+        }
+
+        tmp = dict_new ();
+        if (tmp == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_set_uint64 (tmp, key, fdnum);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
+                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+                        fdnum, fd, uuid_utoa (fd->inode->gfid),
+                        strerror (*op_errno));
+                goto out;
+        }
+
+        len = dict_serialized_length (tmp);
+        if (len < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_serialized_length failed (%s) while handling "
+                        "lockinfo for fd (ptr:%p inode-gfid:%s)",
+                        strerror (*op_errno), fd, uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+        buf = GF_CALLOC (1, len, gf_common_mt_char);
+        if (buf == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_serialize (tmp, buf);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_serialize failed (%s) while handling lockinfo "
+                        "for fd (ptr: %p inode-gfid:%s)", strerror (*op_errno),
+                        fd, uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+        op_ret = dict_set_dynptr (dict, GF_XATTR_LOCKINFO_KEY, buf, len);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
+                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+                        fdnum, fd, uuid_utoa (fd->inode->gfid),
+                        strerror (*op_errno));
+                goto out;
+        }
+
+        buf = NULL;
+out:
+        if (tmp != NULL) {
+                dict_unref (tmp);
+        }
+
+        if (buf != NULL) {
+                GF_FREE (buf);
+        }
+
+        return op_ret;
+}
+
+
+int32_t
+pl_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              const char *name, dict_t *xdata)
+{
+        int32_t  op_ret = 0, op_errno = 0;
+        dict_t  *dict   = NULL;
+
+        if (!name) {
+                goto usual;
+        }
+
+        if (strcmp (name, GF_XATTR_LOCKINFO_KEY) == 0) {
+                dict = dict_new ();
+                if (dict == NULL) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = pl_fgetxattr_handle_lockinfo (this, fd, dict,
+                                                       &op_errno);
+                if (op_ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "getting lockinfo on fd (ptr:%p inode-gfid:%s) "
+                                "failed (%s)", fd, uuid_utoa (fd->inode->gfid),
+                                strerror (op_errno));
+                }
+
+                goto unwind;
+        } else {
+                goto usual;
+        }
+
+unwind:
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, NULL);
+        if (dict != NULL) {
+                dict_unref (dict);
+        }
+
+        return 0;
+
+usual:
+        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+        return 0;
+}
+
+int32_t
+pl_migrate_locks (call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
+                  int32_t *op_errno)
+{
+        pl_inode_t   *pl_inode  = NULL;
+        uint64_t      newfd_num = 0;
+        posix_lock_t *l         = NULL;
+        int32_t       op_ret    = 0;
+
+        newfd_num = fd_to_fdnum (newfd);
+
+        pl_inode = pl_inode_get (frame->this, newfd->inode);
+        if (pl_inode == NULL) {
+                op_ret = -1;
+                *op_errno = EBADFD;
+                goto out;
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (l->fd_num == oldfd_num) {
+                                l->fd_num = newfd_num;
+                                l->client = frame->root->client;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        op_ret = 0;
+out:
+        return op_ret;
+}
+
+int32_t
+pl_fsetxattr_handle_lockinfo (call_frame_t *frame, fd_t *fd, char *lockinfo_buf,
+                              int len, int32_t *op_errno)
+{
+        int32_t   op_ret    = -1;
+        dict_t   *lockinfo  = NULL;
+        uint64_t  oldfd_num = 0;
+        char     *key       = NULL;
+
+        lockinfo = dict_new ();
+        if (lockinfo == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_unserialize (lockinfo_buf, len, &lockinfo);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                goto out;
+        }
+
+        key = pl_lockinfo_key (frame->this, fd->inode, op_errno);
+        if (key == NULL) {
+                op_ret = -1;
+                goto out;
+        }
+
+        op_ret = dict_get_uint64 (lockinfo, key, &oldfd_num);
+
+        if (oldfd_num == 0) {
+                op_ret = 0;
+                goto out;
+        }
+
+        op_ret = pl_migrate_locks (frame, fd, oldfd_num, op_errno);
+        if (op_ret < 0) {
+                gf_log (frame->this->name, GF_LOG_WARNING,
+                        "migration of locks from oldfd (ptr:%p) to newfd "
+                        "(ptr:%p) (inode-gfid:%s)", (void *)oldfd_num, fd,
+                        uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+out:
+        dict_unref (lockinfo);
+
+        return op_ret;
+}
+
+int32_t
+pl_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+        int32_t  op_ret       = 0, op_errno = 0;
+        void    *lockinfo_buf = NULL;
+        int      len          = 0;
+
+        op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                       &lockinfo_buf, &len);
+        if (lockinfo_buf == NULL) {
+                goto usual;
+        }
+
+        op_ret = pl_fsetxattr_handle_lockinfo (frame, fd, lockinfo_buf, len,
+                                               &op_errno);
+        if (op_ret < 0) {
+                goto unwind;
+        }
+
+usual:
+        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+        return 0;
+
+unwind:
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
         return 0;
 }
 
 int32_t
 pl_opendir_cbk (call_frame_t *frame,
-                     void *cookie,
-                     xlator_t *this,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     fd_t *fd)
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                fd_t *fd, dict_t *xdata)
 {
         pl_fdctx_t *fdctx = NULL;
 
@@ -498,28 +928,28 @@ unwind:
                              frame,
                              op_ret,
                              op_errno,
-                             fd);
+                             fd, xdata);
         return 0;
 }
 
 int32_t
 pl_opendir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, fd_t *fd)
+             loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         STACK_WIND (frame,
                     pl_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
-                    loc, fd);
+                    loc, fd, xdata);
         return 0;
 
 }
 
 int
 pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -527,7 +957,7 @@ pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 pl_flush (call_frame_t *frame, xlator_t *this,
-          fd_t *fd)
+          fd_t *fd, dict_t *xdata)
 {
         pl_inode_t *pl_inode = NULL;
 
@@ -535,7 +965,7 @@ pl_flush (call_frame_t *frame, xlator_t *this,
 
         if (!pl_inode) {
                 gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
-                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD);
+                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD, NULL);
                 return 0;
         }
 
@@ -554,7 +984,7 @@ pl_flush (call_frame_t *frame, xlator_t *this,
         }
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                __delete_locks_of_owner (pl_inode, frame->root->trans,
+                __delete_locks_of_owner (pl_inode, frame->root->client,
                                          &frame->root->lk_owner);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
@@ -565,14 +995,14 @@ pl_flush (call_frame_t *frame, xlator_t *this,
 
 wind:
         STACK_WIND (frame, pl_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd);
+                    FIRST_CHILD(this)->fops->flush, fd, xdata);
         return 0;
 }
 
 
 int
 pl_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd)
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         pl_fdctx_t *fdctx = NULL;
 
@@ -587,7 +1017,7 @@ pl_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 unwind:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 
         return 0;
 }
@@ -595,12 +1025,11 @@ unwind:
 
 int
 pl_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+         fd_t *fd, dict_t *xdata)
 {
-        /* why isn't O_TRUNC being handled ? */
         STACK_WIND (frame, pl_open_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
-                    loc, flags & ~O_TRUNC, fd, wbflags);
+                    loc, flags, fd, xdata);
 
         return 0;
 }
@@ -610,7 +1039,7 @@ int
 pl_create_cbk (call_frame_t *frame, void *cookie,
                xlator_t *this, int32_t op_ret, int32_t op_errno,
                fd_t *fd, inode_t *inode, struct iatt *buf,
-               struct iatt *preparent, struct iatt *postparent)
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         pl_fdctx_t *fdctx = NULL;
 
@@ -626,7 +1055,7 @@ pl_create_cbk (call_frame_t *frame, void *cookie,
 
 unwind:
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         return 0;
 }
@@ -634,11 +1063,12 @@ unwind:
 
 int
 pl_create (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+           loc_t *loc, int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+           dict_t *xdata)
 {
         STACK_WIND (frame, pl_create_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
@@ -647,10 +1077,10 @@ int
 pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int32_t op_ret, int32_t op_errno,
               struct iovec *vector, int32_t count, struct iatt *stbuf,
-              struct iobref *iobref)
+              struct iobref *iobref, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref);
+                             vector, count, stbuf, iobref, xdata);
 
         return 0;
 }
@@ -658,9 +1088,10 @@ pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 pl_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 
         return 0;
 }
@@ -718,12 +1149,12 @@ __rw_allowable (pl_inode_t *pl_inode, posix_lock_t *region,
 
 
 int
-pl_readv_cont (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t offset, uint32_t flags)
+pl_readv_cont (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, uint32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, pl_readv_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
 
         return 0;
 }
@@ -731,7 +1162,7 @@ pl_readv_cont (call_frame_t *frame, xlator_t *this,
 
 int
 pl_readv (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, size_t size, off_t offset, uint32_t flags)
+          fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
         pl_inode_t            *pl_inode = NULL;
@@ -748,7 +1179,7 @@ pl_readv (call_frame_t *frame, xlator_t *this,
         if (priv->mandatory && pl_inode->mandatory) {
                 region.fl_start   = offset;
                 region.fl_end     = offset + size - 1;
-                region.transport  = frame->root->trans;
+                region.client     = frame->root->client;
                 region.fd_num     = fd_to_fdnum(fd);
                 region.client_pid = frame->root->pid;
                 region.owner      = frame->root->lk_owner;
@@ -778,7 +1209,8 @@ pl_readv (call_frame_t *frame, xlator_t *this,
                         }
 
                         rw->stub = fop_readv_stub (frame, pl_readv_cont,
-                                                   fd, size, offset, flags);
+                                                   fd, size, offset, flags,
+                                                   xdata);
                         if (!rw->stub) {
                                 op_errno = ENOMEM;
                                 op_ret = -1;
@@ -798,12 +1230,12 @@ pl_readv (call_frame_t *frame, xlator_t *this,
         if (wind_needed) {
                 STACK_WIND (frame, pl_readv_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                            fd, size, offset, flags);
+                            fd, size, offset, flags, xdata);
         }
 
         if (op_ret == -1)
                 STACK_UNWIND_STRICT (readv, frame, -1, op_errno,
-                                     NULL, 0, NULL, NULL);
+                                     NULL, 0, NULL, NULL, NULL);
 
         return 0;
 }
@@ -812,11 +1244,11 @@ pl_readv (call_frame_t *frame, xlator_t *this,
 int
 pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 struct iovec *vector, int count, off_t offset,
-                uint32_t flags, struct iobref *iobref)
+                uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         STACK_WIND (frame, pl_writev_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
 
         return 0;
 }
@@ -825,7 +1257,7 @@ pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int
 pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
            struct iovec *vector, int32_t count, off_t offset,
-           uint32_t flags, struct iobref *iobref)
+           uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
         pl_inode_t            *pl_inode = NULL;
@@ -841,7 +1273,7 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (priv->mandatory && pl_inode->mandatory) {
                 region.fl_start   = offset;
                 region.fl_end     = offset + iov_length (vector, count) - 1;
-                region.transport  = frame->root->trans;
+                region.client     = frame->root->client;
                 region.fd_num     = fd_to_fdnum(fd);
                 region.client_pid = frame->root->pid;
                 region.owner      = frame->root->lk_owner;
@@ -872,7 +1304,7 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
                         rw->stub = fop_writev_stub (frame, pl_writev_cont,
                                                     fd, vector, count, offset,
-                                                    flags, iobref);
+                                                    flags, iobref, xdata);
                         if (!rw->stub) {
                                 op_errno = ENOMEM;
                                 op_ret = -1;
@@ -892,10 +1324,11 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (wind_needed)
                 STACK_WIND (frame, pl_writev_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                            fd, vector, count, offset, flags, iobref);
+                            fd, vector, count, offset, flags, iobref, xdata);
 
         if (op_ret == -1)
-                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL,
+                                     NULL);
 
         return 0;
 }
@@ -921,7 +1354,7 @@ lock_dup (posix_lock_t *lock)
 {
         posix_lock_t *new_lock = NULL;
 
-        new_lock = new_posix_lock (&lock->user_flock, lock->transport,
+        new_lock = new_posix_lock (&lock->user_flock, lock->client,
                                    lock->client_pid, &lock->owner,
                                    (fd_t *)lock->fd_num);
         return new_lock;
@@ -1079,10 +1512,8 @@ unlock:
 
 int
 pl_lk (call_frame_t *frame, xlator_t *this,
-       fd_t *fd, int32_t cmd, struct gf_flock *flock)
+       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        void         *transport  = NULL;
-        pid_t         client_pid = 0;
         pl_inode_t   *pl_inode   = NULL;
         int           op_ret     = 0;
         int           op_errno   = 0;
@@ -1091,9 +1522,6 @@ pl_lk (call_frame_t *frame, xlator_t *this,
         posix_lock_t *conf       = NULL;
         int           ret        = 0;
 
-        transport  = frame->root->trans;
-        client_pid = frame->root->pid;
-
         if ((flock->l_start < 0) || (flock->l_len < 0)) {
                 op_ret = -1;
                 op_errno = EINVAL;
@@ -1107,7 +1535,7 @@ pl_lk (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        reqlock = new_posix_lock (flock, transport, client_pid,
+        reqlock = new_posix_lock (flock, frame->root->client, frame->root->pid,
                                   &frame->root->lk_owner, fd);
 
         if (!reqlock) {
@@ -1225,6 +1653,13 @@ pl_lk (call_frame_t *frame, xlator_t *this,
                         op_ret = -1;
                         op_errno = EAGAIN;
                         __destroy_lock (reqlock);
+
+                } else if ((0 == ret) && (F_UNLCK == flock->l_type)) {
+                        /* For NLM's last "unlock on fd" detection */
+                        if (pl_locks_by_fd (pl_inode, fd))
+                                flock->l_type = F_RDLCK;
+                        else
+                                flock->l_type = F_UNLCK;
                 }
         }
 
@@ -1232,12 +1667,8 @@ unwind:
         pl_trace_out (this, frame, fd, NULL, cmd, flock, op_ret, op_errno, NULL);
         pl_update_refkeeper (this, fd->inode);
 
-        if (pl_locks_by_fd(pl_inode, fd))
-                flock->l_type = F_RDLCK;
-        else
-                flock->l_type = F_UNLCK;
 
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock, xdata);
 out:
         return 0;
 }
@@ -1328,8 +1759,8 @@ pl_forget (xlator_t *this,
                                 list_for_each_entry_safe (entry_l, entry_tmp, &dom->entrylk_list, domain_list) {
                                         list_del_init (&entry_l->domain_list);
 
-                                        if (entry_l->basename)
-                                                GF_FREE ((char *)entry_l->basename);
+                                        GF_FREE ((char *)entry_l->basename);
+                                        GF_FREE (entry_l->connection_id);
                                         GF_FREE (entry_l);
                                 }
 
@@ -1348,21 +1779,22 @@ pl_forget (xlator_t *this,
 
         list_for_each_entry_safe (ext_l, ext_tmp, &posixlks_released, list) {
 
-                STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0, &ext_l->user_flock);
+                STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0,
+                                     &ext_l->user_flock, NULL);
                 __destroy_lock (ext_l);
         }
 
         list_for_each_entry_safe (ino_l, ino_tmp, &inodelks_released, blocked_locks) {
 
-                STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0);
+                STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0, NULL);
                 __pl_inodelk_unref (ino_l);
         }
 
         list_for_each_entry_safe (entry_l, entry_tmp, &entrylks_released, blocked_locks) {
 
-                STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0);
-                if (entry_l->basename)
-                        GF_FREE ((char *)entry_l->basename);
+                STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0, NULL);
+                GF_FREE ((char *)entry_l->basename);
+                GF_FREE (entry_l->connection_id);
                 GF_FREE (entry_l);
 
         }
@@ -1438,7 +1870,7 @@ out:
         return ret;
 }
 
-static int32_t
+int32_t
 __get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
 {
         posix_lock_t *lock   = NULL;
@@ -1446,16 +1878,6 @@ __get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
 
         list_for_each_entry (lock, &pl_inode->ext_list, list) {
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" state: %s",
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len,
-                                lock->blocked == 1 ? "Blocked" : "Active");
-
                 count++;
         }
 
@@ -1488,6 +1910,24 @@ out:
 }
 
 void
+pl_parent_entrylk_xattr_fill (xlator_t *this, inode_t *parent,
+                              char *basename, dict_t *dict)
+{
+        uint32_t         entrylk = 0;
+        int             ret     = -1;
+
+        if (!parent || !basename || !strlen (basename))
+                goto out;
+        entrylk = check_entrylk_on_basename (this, parent, basename);
+out:
+        ret = dict_set_uint32 (dict, GLUSTERFS_PARENT_ENTRYLK, entrylk);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        " dict_set failed on key %s", GLUSTERFS_PARENT_ENTRYLK);
+        }
+}
+
+void
 pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
                        dict_t *dict)
 {
@@ -1504,19 +1944,34 @@ pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
 }
 
 void
-pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode,
-                       dict_t *dict)
+pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode, dict_t *dict,
+                       gf_boolean_t per_dom)
 {
         int32_t     count = 0;
         int         ret   = -1;
+        char        *domname = NULL;
+
+
+        if (per_dom){
+                ret = dict_get_str (dict, GLUSTERFS_INODELK_DOM_COUNT,
+                                    &domname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "value for key %s",GLUSTERFS_INODELK_DOM_COUNT);
+                        goto out;
+                }
+        }
+
+        count = get_inodelk_count (this, inode, domname);
 
-        count = get_inodelk_count (this, inode);
         ret = dict_set_int32 (dict, GLUSTERFS_INODELK_COUNT, count);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_INODELK_COUNT);
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to set count for "
+                        "key %s", GLUSTERFS_INODELK_COUNT);
         }
 
+out:
+        return;
 }
 
 void
@@ -1543,50 +1998,57 @@ pl_lookup_cbk (call_frame_t *frame,
                     int32_t op_errno,
                     inode_t *inode,
                     struct iatt *buf,
-                    dict_t *dict,
+                    dict_t *xdata,
                     struct iatt *postparent)
 {
         pl_local_t *local = NULL;
 
         GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
 
-        if (op_ret) {
+        if (op_ret)
                 goto out;
-        }
 
         local = frame->local;
 
+        if (local->parent_entrylk_req)
+                pl_parent_entrylk_xattr_fill (this, local->loc.parent,
+                                              (char*)local->loc.name, xdata);
         if (local->entrylk_count_req)
-                pl_entrylk_xattr_fill (this, inode, dict);
+                pl_entrylk_xattr_fill (this, inode, xdata);
         if (local->inodelk_count_req)
-                pl_inodelk_xattr_fill (this, inode, dict);
+                pl_inodelk_xattr_fill (this, inode, xdata, _gf_false);
+        if (local->inodelk_dom_count_req)
+                pl_inodelk_xattr_fill (this, inode, xdata, _gf_true);
         if (local->posixlk_count_req)
-                pl_posixlk_xattr_fill (this, inode, dict);
+                pl_posixlk_xattr_fill (this, inode, xdata);
 
 
+out:
+        local = frame->local;
         frame->local = NULL;
 
-        if (local != NULL)
+        if (local != NULL) {
+                loc_wipe (&local->loc);
                 mem_put (local);
+        }
 
-out:
         STACK_UNWIND_STRICT (
                      lookup,
                      frame,
-                      op_ret,
-                      op_errno,
-                      inode,
-                      buf,
-                      dict,
-                      postparent);
+                     op_ret,
+                     op_errno,
+                     inode,
+                     buf,
+                     xdata,
+                     postparent);
         return 0;
 }
 
 int32_t
 pl_lookup (call_frame_t *frame,
-                xlator_t *this,
-                loc_t *loc,
-                dict_t *xattr_req)
+           xlator_t *this,
+           loc_t *loc,
+           dict_t *xdata)
 {
         pl_local_t *local  = NULL;
         int         ret    = -1;
@@ -1598,33 +2060,38 @@ pl_lookup (call_frame_t *frame,
         local = mem_get0 (this->local_pool);
         GF_VALIDATE_OR_GOTO (this->name, local, out);
 
-        if (xattr_req) {
-                if (dict_get (xattr_req, GLUSTERFS_ENTRYLK_COUNT))
+        if (xdata) {
+                if (dict_get (xdata, GLUSTERFS_ENTRYLK_COUNT))
                         local->entrylk_count_req = 1;
-                if (dict_get (xattr_req, GLUSTERFS_INODELK_COUNT))
+                if (dict_get (xdata, GLUSTERFS_INODELK_COUNT))
                         local->inodelk_count_req = 1;
-                if (dict_get (xattr_req, GLUSTERFS_POSIXLK_COUNT))
+                if (dict_get (xdata, GLUSTERFS_INODELK_DOM_COUNT))
+                        local->inodelk_dom_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_POSIXLK_COUNT))
                         local->posixlk_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_PARENT_ENTRYLK))
+                        local->parent_entrylk_req = 1;
         }
 
         frame->local = local;
+        loc_copy (&local->loc, loc);
 
         STACK_WIND (frame,
                     pl_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc,
-                    xattr_req);
+                    loc, xdata);
         ret = 0;
 out:
         if (ret == -1)
-                STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL, NULL, NULL, NULL);
+                STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL,
+                                     NULL, NULL, NULL);
 
         return 0;
 }
 int
 pl_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *entries)
+                 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         pl_local_t *local  = NULL;
         gf_dirent_t *entry = NULL;
@@ -1638,14 +2105,18 @@ pl_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->entrylk_count_req)
                         pl_entrylk_xattr_fill (this, entry->inode, entry->dict);
                 if (local->inodelk_count_req)
-                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict);
+                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
+                                               _gf_false);
+                if (local->inodelk_dom_count_req)
+                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
+                                               _gf_true);
                 if (local->posixlk_count_req)
                         pl_posixlk_xattr_fill (this, entry->inode, entry->dict);
         }
 
 unwind:
         frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 
         if (local)
                 mem_put (local);
@@ -1667,6 +2138,8 @@ pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                         local->entrylk_count_req = 1;
                 if (dict_get (dict, GLUSTERFS_INODELK_COUNT))
                         local->inodelk_count_req = 1;
+                if (dict_get (dict, GLUSTERFS_INODELK_DOM_COUNT))
+                        local->inodelk_dom_count_req = 1;
                 if (dict_get (dict, GLUSTERFS_POSIXLK_COUNT))
                         local->posixlk_count_req = 1;
         }
@@ -1679,17 +2152,19 @@ pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
         return 0;
 out:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL);
+        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
         return 0;
 }
 
 
 void
 pl_dump_lock (char *str, int size, struct gf_flock *flock,
-              gf_lkowner_t *owner, void *trans, time_t *granted_time,
-              time_t *blkd_time, gf_boolean_t active)
+              gf_lkowner_t *owner, void *trans, char *conn_id,
+              time_t *granted_time, time_t *blkd_time, gf_boolean_t active)
 {
-        char *type_str = NULL;
+        char  *type_str    = NULL;
+        char   granted[32] = {0,};
+        char   blocked[32] = {0,};
 
         switch (flock->l_type) {
         case F_RDLCK:
@@ -1713,16 +2188,17 @@ pl_dump_lock (char *str, int size, struct gf_flock *flock,
                                   (unsigned long long) flock->l_start,
                                   (unsigned long long) flock->l_len,
                                   (unsigned long long) flock->l_pid,
-                                  lkowner_utoa (owner),
-                                  trans, ctime (granted_time));
+                                  lkowner_utoa (owner), trans, conn_id,
+                                  ctime_r (granted_time, granted));
                 } else {
                         snprintf (str, size, RANGE_BLKD_GRNTD_FMT,
                                   type_str, flock->l_whence,
                                   (unsigned long long) flock->l_start,
                                   (unsigned long long) flock->l_len,
                                   (unsigned long long) flock->l_pid,
-                                  lkowner_utoa (owner),
-                                  trans, ctime (blkd_time), ctime (granted_time));
+                                  lkowner_utoa (owner), trans, conn_id,
+                                  ctime_r (blkd_time, blocked),
+                                  ctime_r (granted_time, granted));
                 }
         }
         else {
@@ -1731,8 +2207,8 @@ pl_dump_lock (char *str, int size, struct gf_flock *flock,
                           (unsigned long long) flock->l_start,
                           (unsigned long long) flock->l_len,
                           (unsigned long long) flock->l_pid,
-                          lkowner_utoa (owner),
-                          trans, ctime (blkd_time));
+                          lkowner_utoa (owner), trans, conn_id,
+                          ctime_r (blkd_time, blocked));
         }
 
 }
@@ -1742,8 +2218,10 @@ __dump_entrylks (pl_inode_t *pl_inode)
 {
         pl_dom_list_t   *dom  = NULL;
         pl_entry_lock_t *lock = NULL;
-        int             count = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
+        char             blocked[32] = {0,};
+        char             granted[32] = {0,};
+        int              count = 0;
+        char             key[GF_DUMP_MAX_BUF_LEN] = {0,};
 
         char tmp[256];
 
@@ -1766,16 +2244,18 @@ __dump_entrylks (pl_inode_t *pl_inode)
                                           lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
                                           "ENTRYLK_WRLCK", lock->basename,
                                           (unsigned long long) lock->client_pid,
-                                          lkowner_utoa (&lock->owner), lock->trans,
-                                          ctime (&lock->granted_time.tv_sec));
+                                          lkowner_utoa (&lock->owner), lock->client,
+                                          lock->connection_id,
+                                          ctime_r (&lock->granted_time.tv_sec, granted));
                         } else {
                                 snprintf (tmp, 256, ENTRY_BLKD_GRNTD_FMT,
                                           lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
                                           "ENTRYLK_WRLCK", lock->basename,
                                           (unsigned long long) lock->client_pid,
-                                          lkowner_utoa (&lock->owner), lock->trans,
-                                          ctime (&lock->blkd_time.tv_sec),
-                                          ctime (&lock->granted_time.tv_sec));
+                                          lkowner_utoa (&lock->owner), lock->client,
+                                          lock->connection_id,
+                                          ctime_r (&lock->blkd_time.tv_sec, blocked),
+                                          ctime_r (&lock->granted_time.tv_sec, granted));
                         }
 
                         gf_proc_dump_write(key, tmp);
@@ -1792,8 +2272,9 @@ __dump_entrylks (pl_inode_t *pl_inode)
                                   lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
                                   "ENTRYLK_WRLCK", lock->basename,
                                   (unsigned long long) lock->client_pid,
-                                  lkowner_utoa (&lock->owner), lock->trans,
-                                  ctime (&lock->blkd_time.tv_sec));
+                                  lkowner_utoa (&lock->owner), lock->client,
+                                  lock->connection_id,
+                                  ctime_r (&lock->blkd_time.tv_sec, blocked));
 
                         gf_proc_dump_write(key, tmp);
 
@@ -1843,7 +2324,7 @@ __dump_inodelks (pl_inode_t *pl_inode)
                         SET_FLOCK_PID (&lock->user_flock, lock);
                         pl_dump_lock (tmp, 256, &lock->user_flock,
                                       &lock->owner,
-                                      lock->transport,
+                                      lock->client, lock->connection_id,
                                       &lock->granted_time.tv_sec,
                                       &lock->blkd_time.tv_sec,
                                       _gf_true);
@@ -1860,7 +2341,7 @@ __dump_inodelks (pl_inode_t *pl_inode)
                         SET_FLOCK_PID (&lock->user_flock, lock);
                         pl_dump_lock (tmp, 256, &lock->user_flock,
                                       &lock->owner,
-                                      lock->transport,
+                                      lock->client, lock->connection_id,
                                       0, &lock->blkd_time.tv_sec,
                                       _gf_false);
                         gf_proc_dump_write(key, tmp);
@@ -1901,16 +2382,13 @@ __dump_posixlks (pl_inode_t *pl_inode)
                                      count,
                                      lock->blocked ? "BLOCKED" : "ACTIVE");
               pl_dump_lock (tmp, 256, &lock->user_flock,
-                            &lock->owner, lock->transport,
+                            &lock->owner, lock->client, NULL,
                             &lock->granted_time.tv_sec, &lock->blkd_time.tv_sec,
                             (lock->blocked)? _gf_false: _gf_true);
               gf_proc_dump_write(key, tmp);
 
               count++;
         }
-
-
-
 }
 
 void
@@ -1932,24 +2410,36 @@ pl_dump_inode_priv (xlator_t *this, inode_t *inode)
         uint64_t        tmp_pl_inode = 0;
         pl_inode_t      *pl_inode = NULL;
         char            *pathname = NULL;
+        gf_boolean_t    section_added = _gf_false;
 
         int count      = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
+        if (!inode) {
+                errno = EINVAL;
+                goto out;
+        }
 
-        if (ret != 0)
+        ret = TRY_LOCK (&inode->lock);
+        if (ret)
+                goto out;
+        {
+                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
+                if (ret)
+                        goto unlock;
+        }
+unlock:
+        UNLOCK (&inode->lock);
+        if (ret)
                 goto out;
 
         pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-
         if (!pl_inode) {
                 ret = -1;
                 goto out;
         }
 
         gf_proc_dump_add_section("xlator.features.locks.%s.inode", this->name);
+        section_added = _gf_true;
 
         /*We are safe to call __inode_path since we have the
          * inode->table->lock */
@@ -1959,28 +2449,41 @@ pl_dump_inode_priv (xlator_t *this, inode_t *inode)
 
         gf_proc_dump_write("mandatory", "%d", pl_inode->mandatory);
 
-        count = get_entrylk_count (this, inode);
-        if (count) {
-                gf_proc_dump_write("entrylk-count", "%d", count);
-                dump_entrylks(pl_inode);
-        }
+        ret = pthread_mutex_trylock (&pl_inode->mutex);
+        if (ret)
+                goto out;
+        {
+                count = __get_entrylk_count (this, pl_inode);
+                if (count) {
+                        gf_proc_dump_write("entrylk-count", "%d", count);
+                        __dump_entrylks (pl_inode);
+                }
 
-        count = get_inodelk_count (this, inode);
-        if (count) {
-                gf_proc_dump_write("inodelk-count", "%d", count);
-                dump_inodelks(pl_inode);
-        }
+                count = __get_inodelk_count (this, pl_inode, NULL);
+                if (count) {
+                        gf_proc_dump_write("inodelk-count", "%d", count);
+                        __dump_inodelks (pl_inode);
+                }
 
-        count = get_posixlk_count (this, inode);
-        if (count) {
-                gf_proc_dump_write("posixlk-count", "%d", count);
-                dump_posixlks(pl_inode);
+                count = __get_posixlk_count (this, pl_inode);
+                if (count) {
+                        gf_proc_dump_write("posixlk-count", "%d", count);
+                        __dump_posixlks (pl_inode);
+                }
         }
+        pthread_mutex_unlock (&pl_inode->mutex);
 
 out:
-        if (pathname)
-                GF_FREE (pathname);
+        GF_FREE (pathname);
 
+        if (ret && inode) {
+                if (!section_added)
+                        gf_proc_dump_add_section ("xlator.features.locks.%s."
+                                                  "inode", this->name);
+                gf_proc_dump_write ("Unable to print lock state", "(Lock "
+                                    "acquisition failure) %s",
+                                    uuid_utoa (inode->gfid));
+        }
         return ret;
 }
 
@@ -2003,6 +2506,79 @@ mem_acct_init (xlator_t *this)
         return ret;
 }
 
+
+pl_ctx_t*
+pl_ctx_get (client_t *client, xlator_t *xlator)
+{
+        void *tmp = NULL;
+        pl_ctx_t *ctx = NULL;
+
+        client_ctx_get (client, xlator, &tmp);
+
+        ctx = tmp;
+
+        if (ctx != NULL)
+                goto out;
+
+        ctx = GF_CALLOC (1, sizeof (pl_ctx_t), gf_locks_mt_posix_lock_t);
+
+        if (ctx == NULL)
+                goto out;
+
+        pthread_mutex_init (&ctx->lock, NULL);
+	INIT_LIST_HEAD (&ctx->inodelk_lockers);
+	INIT_LIST_HEAD (&ctx->entrylk_lockers);
+
+        if (client_ctx_set (client, xlator, ctx) != 0) {
+                pthread_mutex_destroy (&ctx->lock);
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+out:
+        return ctx;
+}
+
+
+static int
+pl_client_disconnect_cbk (xlator_t *this, client_t *client)
+{
+        pl_ctx_t *pl_ctx = NULL;
+
+	pl_ctx = pl_ctx_get (client, this);
+
+	pl_inodelk_client_cleanup (this, pl_ctx);
+
+	pl_entrylk_client_cleanup (this, pl_ctx);
+
+	return 0;
+}
+
+
+static int
+pl_client_destroy_cbk (xlator_t *this, client_t *client)
+{
+        void     *tmp    = NULL;
+        pl_ctx_t *pl_ctx = NULL;
+
+	pl_client_disconnect_cbk (this, client);
+
+        client_ctx_del (client, this, &tmp);
+
+        if (tmp == NULL)
+                return 0;
+
+        pl_ctx = tmp;
+
+	GF_ASSERT (list_empty(&pl_ctx->inodelk_lockers));
+	GF_ASSERT (list_empty(&pl_ctx->entrylk_lockers));
+
+	pthread_mutex_destroy (&pl_ctx->lock);
+        GF_FREE (pl_ctx);
+
+        return 0;
+}
+
+
 int
 init (xlator_t *this)
 {
@@ -2031,7 +2607,7 @@ init (xlator_t *this)
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "'locks' translator is not loaded over a storage "
                         "translator");
-                goto out;;
+                goto out;
         }
 
         priv = GF_CALLOC (1, sizeof (*priv),
@@ -2065,8 +2641,7 @@ init (xlator_t *this)
 
 out:
         if (ret) {
-                if (priv)
-                        GF_FREE (priv);
+                GF_FREE (priv);
         }
         return ret;
 }
@@ -2081,6 +2656,7 @@ fini (xlator_t *this)
         if (!priv)
                 return 0;
         this->private = NULL;
+        GF_FREE (priv->brickname);
         GF_FREE (priv);
 
         return 0;
@@ -2089,21 +2665,23 @@ fini (xlator_t *this)
 
 int
 pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock);
+            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
+            dict_t *xdata);
 
 int
 pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock);
+             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+             dict_t *xdata);
 
 int
 pl_entrylk (call_frame_t *frame, xlator_t *this,
             const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type);
+            entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 int
 pl_fentrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type);
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 struct xlator_fops fops = {
         .lookup      = pl_lookup,
@@ -2122,6 +2700,8 @@ struct xlator_fops fops = {
         .opendir     = pl_opendir,
         .readdirp    = pl_readdirp,
         .getxattr    = pl_getxattr,
+        .fgetxattr   = pl_fgetxattr,
+        .fsetxattr   = pl_fsetxattr,
 };
 
 struct xlator_dumpops dumpops = {
@@ -2129,9 +2709,11 @@ struct xlator_dumpops dumpops = {
 };
 
 struct xlator_cbks cbks = {
-        .forget      = pl_forget,
-        .release     = pl_release,
-        .releasedir  = pl_releasedir,
+        .forget            = pl_forget,
+        .release           = pl_release,
+        .releasedir        = pl_releasedir,
+        .client_destroy    = pl_client_destroy_cbk,
+        .client_disconnect = pl_client_disconnect_cbk,
 };
 
 
diff --git a/xlators/features/locks/src/reservelk.c b/xlators/features/locks/src/reservelk.c
index 7a75cdea1..11abd26d8 100644
--- a/xlators/features/locks/src/reservelk.c
+++ b/xlators/features/locks/src/reservelk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -299,7 +289,8 @@ grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode)
                         lock->user_flock.l_start,
                         lock->user_flock.l_len);
 
-                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock,
+                                     NULL);
         }
 
 }
@@ -376,7 +367,9 @@ grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode)
                                 pl_trace_out (this, lock->frame, fd, NULL, cmd,
                                               &lock->user_flock, -1, EAGAIN, NULL);
                                 pl_update_refkeeper (this, fd->inode);
-                                STACK_UNWIND_STRICT (lk, lock->frame, -1, EAGAIN, &lock->user_flock);
+                                STACK_UNWIND_STRICT (lk, lock->frame, -1,
+                                                     EAGAIN, &lock->user_flock,
+                                                     NULL);
                                 __destroy_lock (lock);
                         }
                 }
diff --git a/xlators/features/locks/tests/unit-test.c b/xlators/features/locks/tests/unit-test.c
index e95612ad4..d2cca32de 100644
--- a/xlators/features/locks/tests/unit-test.c
+++ b/xlators/features/locks/tests/unit-test.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/features/mac-compat/src/Makefile.am b/xlators/features/mac-compat/src/Makefile.am
index 915c13e30..f8567edce 100644
--- a/xlators/features/mac-compat/src/Makefile.am
+++ b/xlators/features/mac-compat/src/Makefile.am
@@ -1,13 +1,14 @@
 xlator_LTLIBRARIES = mac-compat.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-mac_compat_la_LDFLAGS = -module -avoidversion
+mac_compat_la_LDFLAGS = -module -avoid-version
 
 mac_compat_la_SOURCES = mac-compat.c
 mac_compat_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/mac-compat/src/mac-compat.c b/xlators/features/mac-compat/src/mac-compat.c
index 188b593e4..7cb550ad5 100644
--- a/xlators/features/mac-compat/src/mac-compat.c
+++ b/xlators/features/mac-compat/src/mac-compat.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -57,7 +47,8 @@ static int32_t apple_xattr_len[] = {
 
 int32_t
 maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
         intptr_t ax = (intptr_t)this->private;
         int i = 0;
@@ -80,7 +71,7 @@ maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
          }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
@@ -88,7 +79,7 @@ maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -109,14 +100,14 @@ maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         STACK_WIND (frame, maccomp_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 }
 
 
 int32_t
 maccomp_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name)
+                   const char *name, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -137,21 +128,21 @@ maccomp_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         STACK_WIND (frame, maccomp_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name);
+                    fd, name, xdata);
         return 0;
 }
 
 
 int32_t
 maccomp_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         intptr_t ax = (intptr_t)this->private;
 
         if (op_ret == -1 && ax != GF_XATTR_NONE)
                 op_ret = op_errno = 0;
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -159,7 +150,7 @@ maccomp_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags)
+                  int32_t flags, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -177,14 +168,14 @@ maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
         STACK_WIND (frame, maccomp_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 }
 
 
 int32_t
 maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -202,7 +193,7 @@ maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
         STACK_WIND (frame, maccomp_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, dict, flags, xdata);
         return 0;
 }
 
@@ -239,8 +230,7 @@ struct xlator_fops fops = {
         .fsetxattr   = maccomp_fsetxattr,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
         { .key = {NULL} },
diff --git a/xlators/features/marker/Makefile.am b/xlators/features/marker/Makefile.am
index a6ba2de16..a985f42a8 100644
--- a/xlators/features/marker/Makefile.am
+++ b/xlators/features/marker/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = src @SYNCDAEMON_SUBDIR@
+SUBDIRS = src
 
 CLEANFILES =
diff --git a/xlators/features/marker/src/Makefile.am b/xlators/features/marker/src/Makefile.am
index 501586a76..a7c676472 100644
--- a/xlators/features/marker/src/Makefile.am
+++ b/xlators/features/marker/src/Makefile.am
@@ -1,15 +1,17 @@
 xlator_LTLIBRARIES = marker.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-marker_la_LDFLAGS = -module -avoidversion
+marker_la_LDFLAGS = -module -avoid-version
 
 marker_la_SOURCES = marker.c marker-quota.c marker-quota-helper.c marker-common.c
 marker_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = marker-mem-types.h marker.h marker-quota.h marker-quota-helper.h marker-common.h $(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -fno-strict-aliasing -D$(GF_HOST_OS) \
-        -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src $(GF_CFLAGS) -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES =
 
diff --git a/xlators/features/marker/src/marker-common.c b/xlators/features/marker/src/marker-common.c
index a413781bc..84a718add 100644
--- a/xlators/features/marker/src/marker-common.c
+++ b/xlators/features/marker/src/marker-common.c
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -69,18 +60,10 @@ unlock: UNLOCK (&inode->lock);
         return ret;
 }
 
-void
+int
 marker_filter_quota_xattr (dict_t *dict, char *key,
                            data_t *value, void *data)
 {
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", dict, out);
-        GF_VALIDATE_OR_GOTO ("marker", key, out);
-
-        ret = fnmatch ("trusted.glusterfs.quota*", key, 0);
-        if (ret == 0)
-                dict_del (dict, key);
-out:
-        return;
+        dict_del (dict, key);
+        return 0;
 }
diff --git a/xlators/features/marker/src/marker-common.h b/xlators/features/marker/src/marker-common.h
index 0a7ee2619..23dd846cb 100644
--- a/xlators/features/marker/src/marker-common.h
+++ b/xlators/features/marker/src/marker-common.h
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _MARKER_COMMON_H
 #define _MARKER_COMMON_H
 
@@ -31,6 +22,6 @@
 int32_t
 marker_force_inode_ctx_get (inode_t *, xlator_t *, marker_inode_ctx_t **);
 
-void
+int
 marker_filter_quota_xattr (dict_t *, char *, data_t *, void *);
 #endif
diff --git a/xlators/features/marker/src/marker-mem-types.h b/xlators/features/marker/src/marker-mem-types.h
index 2f49c0d9d..1f74d5048 100644
--- a/xlators/features/marker/src/marker-mem-types.h
+++ b/xlators/features/marker/src/marker-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MARKER_MEM_TYPES_H__
 #define __MARKER_MEM_TYPES_H__
 
diff --git a/xlators/features/marker/src/marker-quota-helper.c b/xlators/features/marker/src/marker-quota-helper.c
index 8d4ff7786..ec0d83316 100644
--- a/xlators/features/marker/src/marker-quota-helper.c
+++ b/xlators/features/marker/src/marker-quota-helper.c
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -163,17 +154,28 @@ out:
 
 
 inode_contribution_t *
-__mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
+__mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx,
+                                loc_t *loc)
 {
-        int32_t ret = 0;
+        int32_t               ret          = 0;
         inode_contribution_t *contribution = NULL;
 
-        if (!loc->parent)
-                goto out;
+        if (!loc->parent) {
+                if (!uuid_is_null (loc->pargfid))
+                        loc->parent = inode_find (loc->inode->table,
+                                                  loc->pargfid);
+
+                if (!loc->parent)
+                        loc->parent = inode_parent (loc->inode, loc->pargfid,
+                                                    loc->name);
+                if (!loc->parent)
+                        goto out;
+        }
 
-        list_for_each_entry (contribution, &ctx->contribution_head, contri_list) {
+        list_for_each_entry (contribution, &ctx->contribution_head,
+                             contri_list) {
                 if (loc->parent &&
-                     uuid_compare (contribution->gfid, loc->parent->gfid) == 0) {
+                    uuid_compare (contribution->gfid, loc->parent->gfid) == 0) {
                         goto out;
                 }
         }
@@ -197,14 +199,16 @@ out:
 
 
 inode_contribution_t *
-mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
+mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx,
+                              loc_t *loc)
 {
         inode_contribution_t *contribution = NULL;
 
         if ((ctx == NULL) || (loc == NULL))
                 return NULL;
 
-        if (strcmp (loc->path, "/") == 0)
+        if (((loc->path) && (strcmp (loc->path, "/") == 0))
+            || (!loc->path && uuid_is_null (loc->pargfid)))
                 return NULL;
 
         LOCK (&ctx->lock);
@@ -227,12 +231,16 @@ mq_dict_set_contribution (xlator_t *this, dict_t *dict,
         GF_VALIDATE_OR_GOTO ("marker", this, out);
         GF_VALIDATE_OR_GOTO ("marker", dict, out);
         GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->parent, out);
 
-        GET_CONTRI_KEY (contri_key, loc->parent->gfid, ret);
-        if (ret < 0) {
-                ret = -1;
-                goto out;
+        if (loc->parent) {
+                GET_CONTRI_KEY (contri_key, loc->parent->gfid, ret);
+                if (ret < 0) {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                /* nameless lookup, fetch contributions to all parents */
+                GET_CONTRI_KEY (contri_key, NULL, ret);
         }
 
         ret = dict_set_int64 (dict, contri_key, 0);
diff --git a/xlators/features/marker/src/marker-quota-helper.h b/xlators/features/marker/src/marker-quota-helper.h
index e019fbd52..6cdd14881 100644
--- a/xlators/features/marker/src/marker-quota-helper.h
+++ b/xlators/features/marker/src/marker-quota-helper.h
@@ -1,20 +1,13 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
+
 #ifndef _MARKER_QUOTA_HELPER_H
 #define _MARKER_QUOTA_HELPER
 
diff --git a/xlators/features/marker/src/marker-quota.c b/xlators/features/marker/src/marker-quota.c
index 43e4ec724..a758e938f 100644
--- a/xlators/features/marker/src/marker-quota.c
+++ b/xlators/features/marker/src/marker-quota.c
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -39,7 +30,8 @@ mq_loc_copy (loc_t *dst, loc_t *src)
         GF_VALIDATE_OR_GOTO ("marker", src, out);
 
         if (src->inode == NULL ||
-            src->path == NULL) {
+            ((src->parent == NULL) && (uuid_is_null (src->pargfid))
+             && !__is_root_gfid (src->inode->gfid))) {
                 gf_log ("marker", GF_LOG_WARNING,
                         "src loc is not valid");
                 goto out;
@@ -208,7 +200,7 @@ out:
 
 int32_t
 mq_dirty_inode_updation_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno)
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         QUOTA_STACK_DESTROY (frame, this);
 
@@ -217,7 +209,7 @@ mq_dirty_inode_updation_done (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno)
+                                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         struct gf_flock   lock  = {0, };
         quota_local_t    *local = NULL;
@@ -229,7 +221,7 @@ mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *thi
         if (op_ret == -1) {
                 local->err = -1;
 
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0);
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
 
                 return 0;
         }
@@ -247,7 +239,7 @@ mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *thi
 	if (ret == -1) {
                 local->err = -1;
                 frame->local = NULL;
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0);
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
                 return 0;
         }
 
@@ -261,20 +253,21 @@ mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *thi
                     mq_dirty_inode_updation_done,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &loc, F_SETLKW, &lock);
+                    this->name, &loc, F_SETLKW, &lock, NULL);
 
         loc_wipe (&loc);
 
         return 0;
 out:
-        mq_dirty_inode_updation_done (frame, NULL, this, -1, 0);
+        mq_dirty_inode_updation_done (frame, NULL, this, -1, 0, NULL);
 
         return 0;
 }
 
 int32_t
 mq_mark_inode_undirty (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
         int32_t        ret     = -1;
         int64_t       *size    = NULL;
@@ -315,14 +308,14 @@ wind:
         STACK_WIND (frame, mq_release_lock_on_dirty_inode,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    &local->loc, newdict, 0);
+                    &local->loc, newdict, 0, NULL);
         ret = 0;
 
 err:
         if (op_ret == -1 || ret == -1) {
                 local->err = -1;
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -379,13 +372,13 @@ mq_update_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto err;
 
         if (uuid_is_null (local->loc.gfid))
-                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
 
         GF_UUID_ASSERT (local->loc.gfid);
 
         STACK_WIND (frame, mq_mark_inode_undirty, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop, &local->loc,
-                    GF_XATTROP_ADD_ARRAY64, new_dict);
+                    GF_XATTROP_ADD_ARRAY64, new_dict, NULL);
 
         ret = 0;
 
@@ -393,7 +386,7 @@ err:
         if (op_ret == -1 || ret == -1) {
                 local->err = -1;
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 
         if (new_dict)
@@ -457,7 +450,7 @@ err:
         if (ret) {
                 local->err = -1;
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 
         if (dict)
@@ -495,7 +488,8 @@ mq_get_child_contribution (call_frame_t *frame,
                 val = -2;
                 if (!mq_test_and_set_local_err (local, &val) &&
                     val != -2)
-                        mq_release_lock_on_dirty_inode (local->frame, NULL, this, 0, 0);
+                        mq_release_lock_on_dirty_inode (local->frame, NULL,
+                                                        this, 0, 0, NULL);
 
                 goto exit;
         }
@@ -523,7 +517,7 @@ out:
 
         if (val == 0) {
                 mq_dirty_inode_readdir (local->frame, NULL, this,
-                                                   0, 0, NULL);
+                                        0, 0, NULL, NULL);
         }
         mq_local_unref (this, local);
 
@@ -539,7 +533,7 @@ mq_readdir_cbk (call_frame_t *frame,
                 xlator_t *this,
                 int32_t op_ret,
                 int32_t op_errno,
-                gf_dirent_t *entries)
+                gf_dirent_t *entries, dict_t *xdata)
 {
         char           contri_key [512]   = {0, };
         int32_t        ret                = 0;
@@ -559,7 +553,7 @@ mq_readdir_cbk (call_frame_t *frame,
                         "readdir failed %s", strerror (op_errno));
                 local->err = -1;
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
 
                 goto end;
         } else if (op_ret == 0) {
@@ -681,7 +675,7 @@ mq_readdir_cbk (call_frame_t *frame,
         }
 
         if (ret && val != -2) {
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 end:
         mq_local_unref (this, local);
@@ -695,7 +689,7 @@ mq_dirty_inode_readdir (call_frame_t *frame,
                         xlator_t *this,
                         int32_t op_ret,
                         int32_t op_errno,
-                        fd_t *fd)
+                        fd_t *fd, dict_t *xdata)
 {
         quota_local_t *local = NULL;
 
@@ -703,7 +697,7 @@ mq_dirty_inode_readdir (call_frame_t *frame,
 
         if (op_ret == -1) {
                 local->err = -1;
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
                 return 0;
         }
 
@@ -714,7 +708,7 @@ mq_dirty_inode_readdir (call_frame_t *frame,
                     mq_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
-                    local->fd, READDIR_BUF, local->d_off);
+                    local->fd, READDIR_BUF, local->d_off, xdata);
 
         return 0;
 }
@@ -754,7 +748,7 @@ mq_check_if_still_dirty (call_frame_t *frame,
 
         //the inode is not dirty anymore
         if (dirty == 0) {
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
 
                 return 0;
         }
@@ -764,21 +758,21 @@ mq_check_if_still_dirty (call_frame_t *frame,
         local->d_off = 0;
 
         if (uuid_is_null (local->loc.gfid))
-                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
 
         GF_UUID_ASSERT (local->loc.gfid);
         STACK_WIND(frame,
                    mq_dirty_inode_readdir,
                    FIRST_CHILD(this),
                    FIRST_CHILD(this)->fops->opendir,
-                   &local->loc, fd);
+                   &local->loc, fd, NULL);
 
         ret = 0;
 
 err:
         if (op_ret == -1 || ret == -1) {
                 local->err = -1;
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
         }
 
         if (fd != NULL) {
@@ -789,15 +783,15 @@ err:
 }
 
 int32_t
-mq_get_dirty_xattr (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int32_t op_ret, int32_t op_errno)
+mq_get_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t        ret       = -1;
         dict_t        *xattr_req = NULL;
         quota_local_t *local     = NULL;
 
         if (op_ret == -1) {
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0);
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
                 return 0;
         }
 
@@ -829,7 +823,7 @@ mq_get_dirty_xattr (call_frame_t *frame, void *cookie,
 err:
         if (ret) {
                 local->err = -1;
-                mq_release_lock_on_dirty_inode(frame, NULL, this, 0, 0);
+                mq_release_lock_on_dirty_inode(frame, NULL, this, 0, 0, NULL);
         }
 
         if (xattr_req)
@@ -896,7 +890,7 @@ mq_update_dirty_inode (xlator_t *this,
                     mq_get_dirty_xattr,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->loc, F_SETLKW, &lock);
+                    this->name, &local->loc, F_SETLKW, &lock, NULL);
         return 1;
 
 fr_destroy:
@@ -909,7 +903,7 @@ out:
 
 int32_t
 mq_inode_creation_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quota_local_t *local = NULL;
 
@@ -931,7 +925,7 @@ mq_inode_creation_done (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
-                                int32_t op_errno)
+                                int32_t op_errno, dict_t *xdata)
 {
         struct gf_flock  lock  = {0, };
         quota_local_t   *local = NULL;
@@ -949,7 +943,7 @@ mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
                     this->name, &local->loc,
-                    F_SETLKW, &lock);
+                    F_SETLKW, &lock, NULL);
 
         return 0;
 }
@@ -957,7 +951,8 @@ mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
 
 int32_t
 mq_create_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
         int32_t          ret       = -1;
         dict_t          *newdict   = NULL;
@@ -986,16 +981,16 @@ mq_create_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
                 STACK_WIND (frame, mq_xattr_creation_release_lock,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->setxattr,
-                            &local->loc, newdict, 0);
+                            &local->loc, newdict, 0, NULL);
         } else {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         ret = 0;
 
 err:
         if (ret < 0) {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict != NULL)
@@ -1044,7 +1039,11 @@ mq_create_xattr (xlator_t *this, call_frame_t *frame)
                         goto free_size;
         }
 
-        if (strcmp (local->loc.path, "/") != 0) {
+        if ((local->loc.path && strcmp (local->loc.path, "/") != 0)
+            || (local->loc.inode && !uuid_is_null (local->loc.inode->gfid) &&
+                !__is_root_gfid (local->loc.inode->gfid))
+            || (!uuid_is_null (local->loc.gfid)
+                && !__is_root_gfid (local->loc.gfid))) {
                 contri = mq_add_new_contribution_node (this, ctx, &local->loc);
                 if (contri == NULL)
                         goto err;
@@ -1061,7 +1060,7 @@ mq_create_xattr (xlator_t *this, call_frame_t *frame)
 
         STACK_WIND (frame, mq_create_dirty_xattr, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop, &local->loc,
-                    GF_XATTROP_ADD_ARRAY64, dict);
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
         ret = 0;
 
 free_size:
@@ -1079,7 +1078,7 @@ err:
 
 out:
         if (ret < 0) {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         return 0;
@@ -1113,7 +1112,12 @@ mq_check_n_set_inode_xattr (call_frame_t *frame, void *cookie,
                 goto create_xattr;
 
         //check contribution xattr if not root
-        if (strcmp (local->loc.path, "/") != 0) {
+        if ((local->loc.path && strcmp (local->loc.path, "/") != 0)
+            || (!uuid_is_null (local->loc.gfid)
+                && !__is_root_gfid (local->loc.gfid))
+            || (local->loc.inode
+                && !uuid_is_null (local->loc.inode->gfid)
+                && !__is_root_gfid (local->loc.inode->gfid))) {
                 GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
                 if (ret < 0)
                         goto out;
@@ -1124,7 +1128,7 @@ mq_check_n_set_inode_xattr (call_frame_t *frame, void *cookie,
         }
 
 out:
-        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 
 create_xattr:
@@ -1139,7 +1143,7 @@ create_xattr:
 
 int32_t
 mq_get_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         dict_t        *xattr_req = NULL;
         quota_local_t *local     = NULL;
@@ -1175,14 +1179,14 @@ mq_get_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 
 err:
-        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0);
+        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
 
         if (xattr_req)
                 dict_unref (xattr_req);
         return 0;
 
 lock_err:
-        mq_inode_creation_done (frame, NULL, this, 0, 0);
+        mq_inode_creation_done (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
@@ -1224,7 +1228,7 @@ mq_set_inode_xattr (xlator_t *this, loc_t *loc)
                     mq_get_xattr,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->loc, F_SETLKW, &lock);
+                    this->name, &local->loc, F_SETLKW, &lock, NULL);
 
         return 0;
 
@@ -1240,6 +1244,7 @@ mq_get_parent_inode_local (xlator_t *this, quota_local_t *local)
 {
         int32_t            ret = -1;
         quota_inode_ctx_t *ctx = NULL;
+        inode_contribution_t *contribution = NULL;
 
         GF_VALIDATE_OR_GOTO ("marker", this, out);
         GF_VALIDATE_OR_GOTO ("marker", local, out);
@@ -1269,7 +1274,7 @@ mq_get_parent_inode_local (xlator_t *this, quota_local_t *local)
         ret = mq_inode_ctx_get (local->loc.inode, this, &ctx);
         if (ret < 0) {
                 gf_log_callingfn (this->name, GF_LOG_WARNING,
-                        "inode ctx get failed");
+                                  "inode ctx get failed");
                 goto out;
         }
 
@@ -1283,7 +1288,31 @@ mq_get_parent_inode_local (xlator_t *this, quota_local_t *local)
                 goto out;
         }
 
-        local->contri = (inode_contribution_t *) ctx->contribution_head.next;
+        /* Earlier we used to get the next entry in the list maintained
+           by the context. In a good situation it works. i.e the next
+           parent in the directory hierarchy for this path is obtained.
+
+           But consider the below situation:
+           mount-point: /mnt/point
+           quota enabled directories within mount point: /a, /b, /c
+
+           Now when some file (file1) in the directory /c is written some data,
+           then to update the directories, marker has to get the contribution
+           object for the parent inode, i.e /c.
+           Beefore, it was being done by
+           local->contri = (inode_contribution_t *) ctx->contribution_head.next;
+           It works in the normal situations. But suppose /c is moved to /b.
+           Now /b's contribution object is added to the end of the list of
+           parents that the file file1 within /b/c is maintaining. Now if
+           the file /b/c/file1 is copied to /b/c/new, to update the parent in
+           the order c, b and / we cannot go to the next element in the list,
+           as in this case the next contribution object would be / and /b's
+           contribution will be at the end of the list. So get the proper
+           parent's contribution, by searching the entire list.
+        */
+        contribution = mq_get_contribution_node (local->loc.parent, ctx);
+        GF_ASSERT (contribution != NULL);
+        local->contri = contribution;
 
         ret = 0;
 out:
@@ -1297,7 +1326,7 @@ mq_xattr_updation_done (call_frame_t *frame,
                         xlator_t *this,
                         int32_t op_ret,
                         int32_t op_errno,
-                        dict_t *dict)
+                        dict_t *dict, dict_t *xdata)
 {
         QUOTA_STACK_DESTROY (frame, this);
         return 0;
@@ -1306,7 +1335,7 @@ mq_xattr_updation_done (call_frame_t *frame,
 
 int32_t
 mq_inodelk_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno)
+                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         ret    = 0;
         gf_boolean_t    status = _gf_false;
@@ -1320,7 +1349,7 @@ mq_inodelk_cbk (call_frame_t *frame, void *cookie,
                                 "unlocking failed on path (%s)(%s)",
                                 local->parent_loc.path, strerror (op_errno));
                 }
-                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL);
+                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL, NULL);
 
                 return 0;
         }
@@ -1330,11 +1359,12 @@ mq_inodelk_cbk (call_frame_t *frame, void *cookie,
 
         if ((strcmp (local->parent_loc.path, "/") == 0)
             || (local->delta == 0)) {
-                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL);
+                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL, NULL);
         } else {
                 ret = mq_get_parent_inode_local (this, local);
                 if (ret < 0) {
-                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL);
+                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL,
+                                                NULL);
                         goto out;
                 }
                 status = _gf_true;
@@ -1343,7 +1373,8 @@ mq_inodelk_cbk (call_frame_t *frame, void *cookie,
                 if (ret == 0 && status == _gf_false) {
                         mq_get_lock_on_parent (frame, this);
                 } else {
-                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL);
+                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL,
+                                                NULL);
                 }
         }
 out:
@@ -1355,7 +1386,7 @@ out:
 int32_t
 mq_release_parent_lock (call_frame_t *frame, void *cookie,
                         xlator_t *this, int32_t op_ret,
-                        int32_t op_errno)
+                        int32_t op_errno, dict_t *xdata)
 {
         int32_t            ret      = 0;
         quota_local_t     *local    = NULL;
@@ -1384,7 +1415,7 @@ mq_release_parent_lock (call_frame_t *frame, void *cookie,
         UNLOCK (&ctx->lock);
 
         if (local->parent_loc.inode == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Invalid parent inode.");
                 goto err;
         }
@@ -1401,12 +1432,12 @@ wind:
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
                     this->name, &local->parent_loc,
-                    F_SETLKW, &lock);
+                    F_SETLKW, &lock, NULL);
 
         return 0;
 err:
         mq_xattr_updation_done (frame, NULL, this,
-                             0, 0 , NULL);
+                                0, 0 , NULL, NULL);
         return 0;
 }
 
@@ -1417,7 +1448,7 @@ mq_mark_undirty (call_frame_t *frame,
                  xlator_t *this,
                  int32_t op_ret,
                  int32_t op_errno,
-                 dict_t *dict)
+                 dict_t *dict, dict_t *xdata)
 {
         int32_t            ret          = -1;
         int64_t           *size         = NULL;
@@ -1478,14 +1509,14 @@ mq_mark_undirty (call_frame_t *frame,
         STACK_WIND (frame, mq_release_parent_lock,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    &local->parent_loc, newdict, 0);
+                    &local->parent_loc, newdict, 0, NULL);
 
         ret = 0;
 err:
         if (op_ret == -1 || ret == -1) {
                 local->err = op_errno;
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -1501,7 +1532,7 @@ mq_update_parent_size (call_frame_t *frame,
                        xlator_t *this,
                        int32_t op_ret,
                        int32_t op_errno,
-                       dict_t *dict)
+                       dict_t *dict, dict_t *xdata)
 {
         int64_t             *size       = NULL;
         int32_t              ret        = -1;
@@ -1525,9 +1556,10 @@ mq_update_parent_size (call_frame_t *frame,
         }
         UNLOCK (&local->contri->lock);
 
-        gf_log (this->name, GF_LOG_DEBUG, "%s %"PRId64 "%"PRId64,
-                local->loc.path, local->ctx->size,
-                local->contri->contribution);
+        gf_log_callingfn (this->name, GF_LOG_DEBUG, "path: %s size: %"PRId64
+                          " contribution:%"PRId64,
+                          local->loc.path, local->ctx->size,
+                          local->contri->contribution);
 
         if (dict == NULL) {
                 op_errno = EINVAL;
@@ -1568,12 +1600,12 @@ mq_update_parent_size (call_frame_t *frame,
                     FIRST_CHILD(this)->fops->xattrop,
                     &local->parent_loc,
                     GF_XATTROP_ADD_ARRAY64,
-                    newdict);
+                    newdict, NULL);
         ret = 0;
 err:
         if (op_ret == -1 || ret < 0) {
                 local->err = op_errno;
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -1655,13 +1687,13 @@ unlock:
         }
         UNLOCK (&contribution->lock);
 
-        gf_log (this->name, GF_LOG_DEBUG, "%s %"PRId64 "%"PRId64,
+        gf_log (this->name, GF_LOG_DEBUG, "%s %"PRId64 " %"PRId64,
                 local->loc.path, size_int, contri_int);
 
         local->delta = size_int - contri_int;
 
         if (local->delta == 0) {
-                mq_mark_undirty (frame, NULL, this, 0, 0, NULL);
+                mq_mark_undirty (frame, NULL, this, 0, 0, NULL, NULL);
                 return 0;
         }
 
@@ -1684,7 +1716,7 @@ unlock:
         }
 
         if (uuid_is_null (local->loc.gfid))
-                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
 
         GF_UUID_ASSERT (local->loc.gfid);
 
@@ -1694,14 +1726,14 @@ unlock:
                     FIRST_CHILD(this)->fops->xattrop,
                     &local->loc,
                     GF_XATTROP_ADD_ARRAY64,
-                    newdict);
+                    newdict, NULL);
         ret = 0;
 
 err:
         if (op_ret == -1 || ret < 0) {
                 local->err = op_errno;
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -1713,7 +1745,7 @@ err:
 int32_t
 mq_fetch_child_size_and_contri (call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
-                                int32_t op_errno)
+                                int32_t op_errno, dict_t *xdata)
 {
         int32_t            ret              = -1;
         char               contri_key [512] = {0, };
@@ -1734,7 +1766,8 @@ mq_fetch_child_size_and_contri (call_frame_t *frame, void *cookie,
         VALIDATE_OR_GOTO (local->ctx, err);
         VALIDATE_OR_GOTO (local->contri, err);
 
-        gf_log (this->name, GF_LOG_DEBUG, "%s marked dirty", local->parent_loc.path);
+        gf_log (this->name, GF_LOG_DEBUG, "%s marked dirty",
+                local->parent_loc.path);
 
         //update parent ctx
         ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
@@ -1795,7 +1828,7 @@ err:
 
                 mq_set_ctx_updation_status (local->ctx, _gf_false);
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (newdict)
@@ -1806,7 +1839,7 @@ err:
 
 int32_t
 mq_markdirty (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno)
+              xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t        ret      = -1;
         dict_t        *dict     = NULL;
@@ -1823,7 +1856,7 @@ mq_markdirty (call_frame_t *frame, void *cookie,
 
                 mq_set_ctx_updation_status (local->ctx, _gf_false);
 
-                mq_inodelk_cbk (frame, NULL, this, 0, 0);
+                mq_inodelk_cbk (frame, NULL, this, 0, 0, NULL);
 
                 return 0;
         }
@@ -1848,7 +1881,7 @@ mq_markdirty (call_frame_t *frame, void *cookie,
         STACK_WIND (frame, mq_fetch_child_size_and_contri,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    &local->parent_loc, dict, 0);
+                    &local->parent_loc, dict, 0, NULL);
 
         ret = 0;
 err:
@@ -1857,7 +1890,7 @@ err:
 
                 mq_set_ctx_updation_status (local->ctx, _gf_false);
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0);
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         if (dict)
@@ -1895,7 +1928,7 @@ mq_get_lock_on_parent (call_frame_t *frame, xlator_t *this)
                     mq_markdirty,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock);
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
 
         return 0;
 
@@ -1905,15 +1938,18 @@ fr_destroy:
         return -1;
 }
 
-
 int
-mq_start_quota_txn (xlator_t *this, loc_t *loc,
-                    quota_inode_ctx_t *ctx,
-                    inode_contribution_t *contri)
+mq_prepare_txn_frame (xlator_t *this, loc_t *loc,
+                      quota_inode_ctx_t *ctx,
+                      inode_contribution_t *contri,
+                      call_frame_t **new_frame)
 {
-        int32_t        ret      = -1;
-        call_frame_t  *frame    = NULL;
-        quota_local_t *local    = NULL;
+        call_frame_t    *frame = NULL;
+        int              ret   = -1;
+        quota_local_t   *local = NULL;
+
+        if (!this || !loc || !new_frame)
+                goto err;
 
         frame = create_frame (this, this->ctx->pool);
         if (frame == NULL)
@@ -1939,14 +1975,36 @@ mq_start_quota_txn (xlator_t *this, loc_t *loc,
         local->ctx = ctx;
         local->contri = contri;
 
+        ret = 0;
+        *new_frame = frame;
+
+        return ret;
+
+fr_destroy:
+        QUOTA_STACK_DESTROY (frame, this);
+err:
+        return ret;
+}
+
+int
+mq_start_quota_txn (xlator_t *this, loc_t *loc,
+                    quota_inode_ctx_t *ctx,
+                    inode_contribution_t *contri)
+{
+        int32_t        ret      = -1;
+        call_frame_t  *frame    = NULL;
+
+        ret = mq_prepare_txn_frame (this, loc, ctx,
+                                    contri, &frame);
+        if (ret)
+                goto err;
+
         ret = mq_get_lock_on_parent (frame, this);
         if (ret == -1)
                 goto err;
 
         return 0;
 
-fr_destroy:
-        QUOTA_STACK_DESTROY (frame, this);
 err:
         mq_set_ctx_updation_status (ctx, _gf_false);
 
@@ -1974,11 +2032,46 @@ mq_initiate_quota_txn (xlator_t *this, loc_t *loc)
                 goto out;
         }
 
+        /* Create the contribution node if its absent. Is it right to
+           assume that if the contribution node is not there, then
+           create one and proceed instead of returning?
+           Reason for this assumption is for hard links. Suppose
+           hard link for a file f1 present in a directory d1 is
+           created in the directory d2 (as f2). Now, since d2's
+           contribution is not there in f1's inode ctx, d2's
+           contribution xattr wont be created and will create problems
+           for quota operations.
+        */
         contribution = mq_get_contribution_node (loc->parent, ctx);
-        if (contribution == NULL)
-                goto out;
+        if (!contribution) {
+                if ((loc->path && strcmp (loc->path, "/"))
+                    || (!uuid_is_null (loc->gfid)
+                        && !__is_root_gfid (loc->gfid))
+                    || (loc->inode && !uuid_is_null (loc->inode->gfid)
+                        && !__is_root_gfid (loc->inode->gfid)))
+                        gf_log_callingfn (this->name, GF_LOG_TRACE,
+                                          "contribution node for the "
+                                          "path (%s) with parent (%s) "
+                                          "not found", loc->path,
+                                          loc->parent?
+                                          uuid_utoa (loc->parent->gfid):
+                                          NULL);
 
-        /* To improve performance, donot start another transaction
+                contribution = mq_add_new_contribution_node (this, ctx, loc);
+                if (!contribution) {
+                        if(loc->path && strcmp (loc->path, "/"))
+                                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                                  "could not allocate "
+                                                  " contribution node for (%s) "
+                                                  "parent: (%s)", loc->path,
+                                                  loc->parent?
+                                                  uuid_utoa (loc->parent->gfid):
+                                                  NULL);
+                        goto out;
+                }
+        }
+
+        /* To improve performance, do not start another transaction
          * if one is already in progress for same inode
          */
         status = _gf_true;
@@ -1997,16 +2090,7 @@ out:
 }
 
 
-/* int32_t */
-/* validate_inode_size_contribution (xlator_t *this, loc_t *loc, int64_t size, */
-/*                                int64_t contribution) */
-/* { */
-/*   if (size != contribution) { */
-/*     mq_initiate_quota_txn (this, loc); */
-/*   } */
 
-/*   return 0; */
-/* } */
 
 
 int32_t
@@ -2035,11 +2119,13 @@ mq_inspect_directory_xattr (xlator_t *this,
                 }
         }
 
-        if (strcmp (loc->path, "/") != 0) {
+        if (!loc->path || (loc->path && strcmp (loc->path, "/") != 0)) {
                 contribution = mq_add_new_contribution_node (this, ctx, loc);
                 if (contribution == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot add a new contribution node");
+                        if (!uuid_is_null (loc->inode->gfid))
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "cannot add a new contribution node "
+                                        "(%s)", uuid_utoa (loc->inode->gfid));
                         ret = -1;
                         goto err;
                 }
@@ -2053,7 +2139,10 @@ mq_inspect_directory_xattr (xlator_t *this,
         if (ret < 0)
                 goto out;
 
-        if (strcmp (loc->path, "/") != 0) {
+        if ((loc->path && strcmp (loc->path, "/") != 0)
+            || (!uuid_is_null (loc->gfid) && !__is_root_gfid (loc->gfid))
+            || (loc->inode && !uuid_is_null (loc->inode->gfid) &&
+                !__is_root_gfid (loc->inode->gfid))) {
                 not_root = _gf_true;
 
                 GET_CONTRI_KEY (contri_key, contribution->gfid, ret);
@@ -2125,8 +2214,11 @@ mq_inspect_file_xattr (xlator_t *this,
         }
 
         contribution = mq_add_new_contribution_node (this, ctx, loc);
-        if (contribution == NULL)
+        if (contribution == NULL) {
+                gf_log_callingfn (this->name, GF_LOG_DEBUG, "cannot allocate "
+                                  "contribution node (path:%s)", loc->path);
                 goto out;
+        }
 
         LOCK (&ctx->lock);
         {
@@ -2158,8 +2250,12 @@ mq_inspect_file_xattr (xlator_t *this,
                         if (size != contri_int) {
                                 mq_initiate_quota_txn (this, loc);
                         }
-                } else
-                        mq_initiate_quota_txn (this, loc);
+                } else {
+                        if (size)
+                                mq_initiate_quota_txn (this, loc);
+                        else
+                                mq_set_inode_xattr (this, loc);
+                }
         }
 
 out:
@@ -2172,8 +2268,8 @@ mq_xattr_state (xlator_t *this,
                 dict_t *dict,
                 struct iatt buf)
 {
-        if (buf.ia_type == IA_IFREG ||
-            buf.ia_type == IA_IFLNK) {
+        if (((buf.ia_type == IA_IFREG) && !dht_is_linkfile (&buf, dict))
+            || (buf.ia_type == IA_IFLNK))  {
                 mq_inspect_file_xattr (this, loc, dict, buf);
         } else if (buf.ia_type == IA_IFDIR)
                 mq_inspect_directory_xattr (this, loc, dict, buf);
@@ -2195,7 +2291,7 @@ mq_req_xattr (xlator_t *this,
                 goto set_size;
 
         //if not "/" then request contribution
-        if (strcmp (loc->path, "/") == 0)
+        if (loc->path && strcmp (loc->path, "/") == 0)
                 goto set_size;
 
         ret = mq_dict_set_contribution (this, dict, loc);
@@ -2224,7 +2320,7 @@ out:
 
 int32_t
 mq_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         QUOTA_STACK_DESTROY (frame, this);
 
@@ -2233,32 +2329,105 @@ mq_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 _mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t        ret                = 0;
         char           contri_key [512]   = {0, };
         quota_local_t *local              = NULL;
+        inode_t       *inode              = NULL;
+        dentry_t      *tmp                = NULL;
+        gf_boolean_t  last_dentry         = _gf_true;
+        loc_t         loc                 = {0, };
+        dentry_t      *other_dentry       = NULL;
+        gf_boolean_t  remove              = _gf_false;
 
         local = (quota_local_t *) frame->local;
 
         if (op_ret == -1 || local->err == -1) {
-                mq_removexattr_cbk (frame, NULL, this, -1, 0);
+                mq_removexattr_cbk (frame, NULL, this, -1, 0, NULL);
                 return 0;
         }
 
         frame->local = NULL;
 
-        if (local->hl_count > 1) {
-                GET_CONTRI_KEY (contri_key, local->contri->gfid, ret);
+        GET_CONTRI_KEY (contri_key, local->contri->gfid, ret);
 
-                STACK_WIND (frame, mq_removexattr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->removexattr,
-                            &local->loc, contri_key);
-                ret = 0;
-        } else {
-                mq_removexattr_cbk (frame, NULL, this, 0, 0);
+        if (!local->loc.inode)
+                inode = inode_grep (local->loc.parent->table, local->loc.parent,
+                                    local->loc.name);
+        else
+                inode = inode_ref (local->loc.inode);
+
+        /* Suppose there are 2 directories dir1 and dir2. Quota limit is set on
+           both the directories. There is a file (f1) in dir1. A hark link is
+           created for that file inside the directory dir2 (say f2). Now one
+           more xattr is set in the inode as a new hard link is created in a
+           separate directory.
+           i.e trusted.glusterfs.quota.<gfid of dir2>.contri=<contribution>
+
+           Now when the hardlink f2 is removed, then the new xattr added (i.e
+           the xattr indicating its contribution to ITS parent directory) should
+           be removed (IFF there is not another hardlink for that file in the
+           same directory).
+
+           To do that upon getting unlink first check whether any other hard
+           links for the same inode exists in the same directory. If so do not
+           do anything and proceed for quota transaction.
+           Otherwise, if the removed entry was the only link for that inode
+           within that directory, then get another dentry for the inode
+           (by traversing the list of dentries for the inode) and using the
+           the dentry's parent and name, send removexattr so that the xattr
+           is removed.
+
+           If it is not done, then if the volume is restarted or the brick
+           process is restarted, then wrong quota usage will be shown for the
+           directory dir2.
+        */
+        if (inode) {
+                tmp = NULL;
+                list_for_each_entry (tmp, &inode->dentry_list, inode_list) {
+                        if (local->loc.parent == tmp->parent) {
+                                if (strcmp (local->loc.name, local->loc.name)) {
+                                        last_dentry = _gf_false;
+                                        break;
+                                }
+                        }
+                }
+                remove = last_dentry;
         }
 
+        if (remove) {
+                if (!other_dentry)    {
+                        list_for_each_entry (tmp, &inode->dentry_list,
+                                             inode_list) {
+                                if (local->loc.parent != tmp->parent) {
+                                        other_dentry = tmp;
+                                        break;
+                                }
+                        }
+                }
+
+                if (!other_dentry)
+                        mq_removexattr_cbk (frame, NULL, this, 0, 0, NULL);
+                else {
+                        loc.parent = inode_ref (other_dentry->parent);
+                        loc.name = gf_strdup (other_dentry->name);
+                        uuid_copy (loc.pargfid , other_dentry->parent->gfid);
+                        loc.inode = inode_ref (inode);
+                        uuid_copy (loc.gfid, inode->gfid);
+                        inode_path (other_dentry->parent, other_dentry->name,
+                                    (char **)&loc.path);
+
+                        STACK_WIND (frame, mq_removexattr_cbk,
+                                    FIRST_CHILD(this),
+                                    FIRST_CHILD(this)->fops->removexattr,
+                                    &loc, contri_key, NULL);
+                }
+        } else
+                mq_removexattr_cbk (frame, NULL, this, 0, 0, NULL);
+
+        ret = 0;
+
         if (strcmp (local->parent_loc.path, "/") != 0) {
                 ret = mq_get_parent_inode_local (this, local);
                 if (ret < 0)
@@ -2269,12 +2438,15 @@ _mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         mq_local_unref (this, local);
 
+        loc_wipe (&loc);
+        inode_unref (inode);
         return 0;
 }
 
 int32_t
 mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
         int32_t            ret   = -1;
         struct gf_flock    lock  = {0, };
@@ -2321,13 +2493,13 @@ mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
                     this->name, &local->parent_loc,
-                    F_SETLKW, &lock);
+                    F_SETLKW, &lock, NULL);
         return 0;
 }
 
 int32_t
-mq_reduce_parent_size_xattr (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno)
+mq_reduce_parent_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t                  ret               = -1;
         int64_t                 *size              = NULL;
@@ -2364,13 +2536,13 @@ mq_reduce_parent_size_xattr (call_frame_t *frame, void *cookie,
 
         STACK_WIND (frame, mq_inode_remove_done, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop, &local->parent_loc,
-                    GF_XATTROP_ADD_ARRAY64, dict);
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
         dict_unref (dict);
         return 0;
 
 err:
         local->err = 1;
-        mq_inode_remove_done (frame, NULL, this, -1, 0, NULL);
+        mq_inode_remove_done (frame, NULL, this, -1, 0, NULL, NULL);
         if (dict)
                 dict_unref (dict);
         return 0;
@@ -2394,8 +2566,11 @@ mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
                 goto out;
 
         contribution = mq_get_contribution_node (loc->parent, ctx);
-        if (contribution == NULL)
+        if (contribution == NULL) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING, "contribution for"
+                                  " the node %s is NULL", loc->path);
                 goto out;
+        }
 
         local = mq_local_new ();
         if (local == NULL) {
@@ -2414,6 +2589,8 @@ mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
         }
 
         if (local->size == 0) {
+                gf_log_callingfn (this->name, GF_LOG_TRACE,
+                                  "local->size is 0 " "path: (%s)", loc->path);
                 ret = 0;
                 goto out;
         }
@@ -2426,8 +2603,12 @@ mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
         local->contri = contribution;
 
         ret = mq_inode_loc_fill (NULL, loc->parent, &local->parent_loc);
-        if (ret < 0)
+        if (ret < 0) {
+                gf_log_callingfn (this->name, GF_LOG_INFO, "building parent loc"
+                                  " failed. (gfid: %s)",
+                                  uuid_utoa (loc->parent->gfid));
                 goto out;
+        }
 
         frame = create_frame (this, this->ctx->pool);
         if (!frame) {
@@ -2446,7 +2627,7 @@ mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
 
         if (local->parent_loc.inode == NULL) {
                 ret = -1;
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Inode is NULL, so can't stackwind.");
                 goto out;
         }
@@ -2455,7 +2636,7 @@ mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
                     mq_reduce_parent_size_xattr,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock);
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
         local = NULL;
         ret = 0;
 
diff --git a/xlators/features/marker/src/marker-quota.h b/xlators/features/marker/src/marker-quota.h
index 30ee44264..42def9d22 100644
--- a/xlators/features/marker/src/marker-quota.h
+++ b/xlators/features/marker/src/marker-quota.h
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _MARKER_QUOTA_H
 #define _MARKER_QUOTA_H
 
@@ -51,8 +42,6 @@
                 var = GF_CALLOC (sizeof (type), 1,      \
                                  gf_marker_mt_##type);  \
                 if (!var) {                             \
-                        gf_log ("", GF_LOG_ERROR,       \
-                                "out of memory");       \
                         ret = -1;                       \
                 }                                       \
         } while (0);
@@ -70,13 +59,20 @@
                 ret = 0;                                \
         } while (0);
 
-#define GET_CONTRI_KEY(var, _gfid, _ret)        \
-        do {                                    \
-                char _gfid_unparsed[40];        \
-                uuid_unparse (_gfid, _gfid_unparsed); \
-                _ret = snprintf (var, CONTRI_KEY_MAX, QUOTA_XATTR_PREFIX \
-                                 ".%s.%s." CONTRIBUTION, "quota", \
-                                 _gfid_unparsed); \
+#define GET_CONTRI_KEY(var, _gfid, _ret)              \
+        do {                                          \
+                if (_gfid != NULL) {                  \
+                        char _gfid_unparsed[40];      \
+                        uuid_unparse (_gfid, _gfid_unparsed);           \
+                        _ret = snprintf (var, CONTRI_KEY_MAX,           \
+                                         QUOTA_XATTR_PREFIX             \
+                                         ".%s.%s." CONTRIBUTION, "quota", \
+                                         _gfid_unparsed);               \
+                } else {                                                \
+                        _ret = snprintf (var, CONTRI_KEY_MAX,           \
+                                         QUOTA_XATTR_PREFIX             \
+                                         ".%s.." CONTRIBUTION, "quota"); \
+                }                                                       \
         } while (0);
 
 #define QUOTA_SAFE_INCREMENT(lock, var)         \
@@ -123,7 +119,7 @@ mq_initiate_quota_txn (xlator_t *, loc_t *);
 
 int32_t
 mq_dirty_inode_readdir (call_frame_t *, void *, xlator_t *,
-                        int32_t, int32_t, fd_t *);
+                        int32_t, int32_t, fd_t *, dict_t *);
 
 int32_t
 mq_reduce_parent_size (xlator_t *, loc_t *, int64_t);
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index e8bbd9d94..a27a266f0 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -30,6 +21,7 @@
 #include "marker-quota-helper.h"
 #include "marker-common.h"
 #include "byte-order.h"
+#include "syncop.h"
 
 #define _GF_UID_GID_CHANGED 1
 
@@ -65,23 +57,25 @@ marker_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 
         if (inode) {
                 loc->inode = inode_ref (inode);
-		uuid_copy (loc->gfid, loc->inode->gfid);
+                if (uuid_is_null (loc->gfid)) {
+                        uuid_copy (loc->gfid, loc->inode->gfid);
+                }
         }
 
         if (parent)
                 loc->parent = inode_ref (parent);
 
-        loc->path = gf_strdup (path);
-        if (!loc->path) {
-                gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
-                goto loc_wipe;
-        }
+        if (path) {
+                loc->path = gf_strdup (path);
+                if (!loc->path) {
+                        gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
+                        goto loc_wipe;
+                }
 
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-        else
-                goto loc_wipe;
+                loc->name = strrchr (loc->path, '/');
+                if (loc->name)
+                        loc->name++;
+        }
 
         ret = 0;
 loc_wipe:
@@ -115,8 +109,7 @@ err:
 	if (parent)
 		inode_unref (parent);
 
-        if (resolvedpath)
-                GF_FREE (resolvedpath);
+        GF_FREE (resolvedpath);
 
         return ret;
 }
@@ -154,12 +147,21 @@ out:
 }
 
 int32_t
-marker_error_handler (xlator_t *this)
+marker_error_handler (xlator_t *this, marker_local_t *local, int32_t op_errno)
 {
-        marker_conf_t *priv = NULL;
+         marker_conf_t *priv = NULL;
+         const char    *path = NULL;
 
-        priv = (marker_conf_t *) this->private;
+         priv = (marker_conf_t *) this->private;
+         path = local
+                 ? (local->loc.path
+                    ? local->loc.path : uuid_utoa(local->loc.gfid))
+                 : "<nul>";
 
+         gf_log (this->name, GF_LOG_CRITICAL,
+                 "Indexing gone corrupt at %s (reason: %s)."
+                 " Geo-replication slave content needs to be revalidated",
+                 path, strerror (op_errno));
         unlink (priv->timestamp_file);
 
         return 0;
@@ -184,6 +186,8 @@ marker_local_unref (marker_local_t *local)
 
         loc_wipe (&local->loc);
         loc_wipe (&local->parent_loc);
+        if (local->xdata)
+                dict_unref (local->xdata);
 
         if (local->oplocal) {
                 marker_local_unref (local->oplocal);
@@ -224,13 +228,14 @@ stat_stampfile (xlator_t *this, marker_conf_t *priv,
 
 int32_t
 marker_getxattr_stampfile_cbk (call_frame_t *frame, xlator_t *this,
-                               const char *name, struct volume_mark *vol_mark)
+                               const char *name, struct volume_mark *vol_mark,
+                               dict_t *xdata)
 {
         int32_t   ret  = -1;
         dict_t   *dict = NULL;
 
         if (vol_mark == NULL){
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
 
                 goto out;
         }
@@ -243,7 +248,7 @@ marker_getxattr_stampfile_cbk (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_WARNING, "failed to set key %s",
                         name);
 
-        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, dict, xdata);
 
         dict_unref (dict);
 out:
@@ -267,42 +272,105 @@ call_from_special_client (call_frame_t *frame, xlator_t *this, const char *name)
 
         stat_stampfile (this, priv, &vol_mark);
 
-        marker_getxattr_stampfile_cbk (frame, this, name, vol_mark);
+        marker_getxattr_stampfile_cbk (frame, this, name, vol_mark, NULL);
 out:
         return ret;
 }
 
 int32_t
 marker_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
+        int            ret    = 0;
+        char           *src   = NULL;
+        char           *dst   = NULL;
+        int            len    = 0;
+        marker_local_t *local = NULL;
+
+        local = frame->local;
+
+
         if (cookie) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Filtering the quota extended attributes");
 
-                dict_foreach (dict, marker_filter_quota_xattr, NULL);
+                /* If the getxattr is from a non special client, then do not
+                   copy the quota related xattrs (except the quota limit key
+                   i.e trusted.glusterfs.quota.limit-set which has been set by
+                   glusterd on the directory on which quota limit is set.) for
+                   directories. Let the healing of xattrs happen upon lookup.
+                   NOTE: setting of trusted.glusterfs.quota.limit-set as of now
+                   happens from glusterd. It should be moved to quotad. Also
+                   trusted.glusterfs.quota.limit-set is set on directory which
+                   is permanent till quota is removed on that directory or limit
+                   is changed. So let that xattr be healed by other xlators
+                   properly whenever directory healing is done.
+                */
+                ret = dict_get_ptr_and_len (dict, QUOTA_LIMIT_KEY,
+                                            (void **)&src, &len);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "dict_get on %s "
+                                "failed", QUOTA_LIMIT_KEY);
+                } else {
+                        dst = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+                        if (dst)
+                                memcpy (dst, src, len);
+                }
+
+                /*
+                 * Except limit-set xattr, rest of the xattrs are maintained
+                 * by quota xlator. Don't expose them to other xlators.
+                 * This filter makes sure quota xattrs are not healed as part of
+                 * metadata self-heal
+                 */
+                GF_REMOVE_INTERNAL_XATTR ("trusted.glusterfs.quota*", dict);
+                if (!ret && IA_ISDIR (local->loc.inode->ia_type) && dst) {
+                        ret = dict_set_dynptr (dict, QUOTA_LIMIT_KEY,
+                                               dst, len);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING, "setting "
+                                        "key %s failed", QUOTA_LIMIT_KEY);
+                        else
+                                dst = NULL;
+                }
         }
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+
+        GF_FREE (dst);
+
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        marker_local_unref (local);
         return 0;
 }
 
 int32_t
 marker_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
         gf_boolean_t   ret    = _gf_false;
         marker_conf_t *priv   = NULL;
         unsigned long  cookie = 0;
+        marker_local_t *local = NULL;
 
         priv = this->private;
 
-        if (priv == NULL || (priv->feature_enabled & GF_XTIME) == 0)
-                goto wind;
+        frame->local = mem_get0 (this->local_pool);
+        local = frame->local;
+        if (local == NULL)
+                goto out;
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret < 0)
+                goto out;
 
         gf_log (this->name, GF_LOG_DEBUG, "USER:PID = %d", frame->root->pid);
 
-        ret = call_from_special_client (frame, this, name);
-wind:
+        if (priv && priv->feature_enabled & GF_XTIME)
+                ret = call_from_special_client (frame, this, name);
+
         if (ret == _gf_false) {
                 if (name == NULL) {
                         /* Signifies that marker translator
@@ -315,10 +383,15 @@ wind:
                 STACK_WIND_COOKIE (frame, marker_getxattr_cbk, (void *)cookie,
                                    FIRST_CHILD(this),
                                    FIRST_CHILD(this)->fops->getxattr, loc,
-                                   name);
+                                   name, xdata);
         }
 
         return 0;
+out:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+        marker_local_unref (local);
+        return 0;
 }
 
 
@@ -340,7 +413,7 @@ marker_setxattr_done (call_frame_t *frame)
 
 int
 marker_specific_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno)
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         ret   = 0;
         int32_t         done  = 0;
@@ -349,14 +422,20 @@ marker_specific_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = (marker_local_t*) frame->local;
 
         if (op_ret == -1 && op_errno == ENOSPC) {
-                marker_error_handler (this);
+                marker_error_handler (this, local, op_errno);
                 done = 1;
                 goto out;
         }
 
-        if (strcmp (local->loc.path, "/") == 0) {
-                done = 1;
-                goto out;
+        if (local) {
+                if (local->loc.path && strcmp (local->loc.path, "/") == 0) {
+                        done = 1;
+                        goto out;
+                }
+                if (__is_root_gfid (local->loc.gfid)) {
+                        done = 1;
+                        goto out;
+                }
         }
 
         ret = marker_trav_parent (local);
@@ -414,7 +493,8 @@ marker_start_setxattr (call_frame_t *frame, xlator_t *this)
         }
 
         STACK_WIND (frame, marker_specific_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0);
+                    FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0,
+                    NULL);
 
         ret = 0;
 out:
@@ -454,10 +534,16 @@ marker_create_frame (xlator_t *this, marker_local_t *local)
 int32_t
 marker_xtime_update_marks (xlator_t *this, marker_local_t *local)
 {
+        marker_conf_t *priv = NULL;
+
         GF_VALIDATE_OR_GOTO ("marker", this, out);
         GF_VALIDATE_OR_GOTO (this->name, local, out);
 
-        if (local->pid == -1)
+        priv = this->private;
+
+        if ((local->pid == GF_CLIENT_PID_GSYNCD
+             && !(priv->feature_enabled & GF_XTIME_GSYNC_FORCE))
+            || (local->pid == GF_CLIENT_PID_DEFRAG))
                 goto out;
 
         marker_gettimeofday (local);
@@ -474,7 +560,7 @@ int32_t
 marker_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -489,11 +575,14 @@ marker_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent);
+                             buf, preparent, postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
         priv = this->private;
 
         if (priv->feature_enabled & GF_QUOTA)
@@ -510,7 +599,7 @@ out:
 
 int
 marker_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dict_t *params)
+              mode_t umask, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -531,12 +620,12 @@ marker_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 goto err;
 wind:
         STACK_WIND (frame, marker_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 
         return 0;
 err:
         STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL);
+                             NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -545,7 +634,7 @@ int32_t
 marker_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -560,11 +649,14 @@ marker_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
         priv = this->private;
 
         if (priv->feature_enabled & GF_QUOTA)
@@ -581,7 +673,7 @@ out:
 
 int32_t
 marker_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-               mode_t mode, fd_t *fd, dict_t *params)
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -602,12 +694,12 @@ marker_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                 goto err;
 wind:
         STACK_WIND (frame, marker_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, fd,
-                    params);
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
         return 0;
 err:
         STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                             NULL);
+                             NULL, NULL);
 
         return 0;
 }
@@ -616,7 +708,7 @@ err:
 int32_t
 marker_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -630,7 +722,8 @@ marker_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -656,7 +749,7 @@ marker_writev (call_frame_t *frame,
                struct iovec *vector,
                int32_t count,
                off_t offset, uint32_t flags,
-               struct iobref *iobref)
+               struct iobref *iobref, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -678,10 +771,10 @@ marker_writev (call_frame_t *frame,
 wind:
         STACK_WIND (frame, marker_writev_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
-                    flags, iobref);
+                    flags, iobref, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -690,7 +783,7 @@ err:
 int32_t
 marker_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -705,7 +798,7 @@ marker_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent);
+                             postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -724,7 +817,8 @@ out:
 }
 
 int32_t
-marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -745,10 +839,10 @@ marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
                 goto err;
 wind:
         STACK_WIND (frame, marker_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags);
+                    FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -757,7 +851,7 @@ err:
 int32_t
 marker_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -772,15 +866,17 @@ marker_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent);
+                             postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
         priv = this->private;
 
-        if ((priv->feature_enabled & GF_QUOTA) && (local->ia_nlink == 1))
-                mq_reduce_parent_size (this, &local->loc, -1);
+        if (priv->feature_enabled & GF_QUOTA) {
+                if (!local->skip_txn)
+                        mq_reduce_parent_size (this, &local->loc, -1);
+        }
 
         if (priv->feature_enabled & GF_XTIME)
                 marker_xtime_update_marks (this, local);
@@ -792,36 +888,8 @@ out:
 
 
 int32_t
-marker_unlink_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        marker_local_t *local = NULL;
-
-        local = frame->local;
-        if (op_ret < 0) {
-                goto err;
-        }
-
-        if (local == NULL) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        local->ia_nlink = buf->ia_nlink;
-
-        STACK_WIND (frame, marker_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, &local->loc);
-        return 0;
-err:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
-        marker_local_unref (local);
-        return 0;
-}
-
-
-int32_t
-marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -833,7 +901,9 @@ marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto unlink_wind;
 
         local = mem_get0 (this->local_pool);
-
+        local->xflag = xflag;
+        if (xdata)
+                local->xdata = dict_ref (xdata);
         MARKER_INIT_LOCAL (frame, local);
 
         ret = loc_copy (&local->loc, loc);
@@ -841,20 +911,18 @@ marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         if (ret == -1)
                 goto err;
 
-        if (uuid_is_null (loc->gfid) && loc->inode)
-                uuid_copy (loc->gfid, loc->inode->gfid);
-
-        STACK_WIND (frame, marker_unlink_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
-        return 0;
+        if (xdata && dict_get (xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY)) {
+                local->skip_txn = 1;
+                goto unlink_wind;
+        }
 
 unlink_wind:
         STACK_WIND (frame, marker_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
         return 0;
 err:
         frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
         marker_local_unref (local);
         return 0;
 }
@@ -864,7 +932,7 @@ int32_t
 marker_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -879,15 +947,18 @@ marker_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
         priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc);
+        if (priv->feature_enabled & GF_QUOTA) {
+                if (!local->skip_txn)
+                        mq_set_inode_xattr (this, &local->loc);
+        }
+
 
         if (priv->feature_enabled & GF_XTIME)
                 marker_xtime_update_marks (this, local);
@@ -898,7 +969,8 @@ out:
 }
 
 int32_t
-marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -917,12 +989,16 @@ marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 
         if (ret == -1)
                 goto err;
+
+        if (xdata && dict_get (xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY))
+                local->skip_txn = 1;
 wind:
         STACK_WIND (frame, marker_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                             NULL);
 
         return 0;
 }
@@ -930,7 +1006,7 @@ err:
 
 int32_t
 marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t *local  = NULL, *oplocal = NULL;
         loc_t           newloc = {0, };
@@ -945,7 +1021,7 @@ marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (op_ret < 0) {
                 if (local->err == 0) {
-                        local->err = op_errno;
+                        local->err =  op_errno ? op_errno : EINVAL;
                 }
 
                 gf_log (this->name, GF_LOG_WARNING,
@@ -960,7 +1036,12 @@ marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->stub = NULL;
         } else if (local->err != 0) {
                 STACK_UNWIND_STRICT (rename, frame, -1, local->err, NULL, NULL,
-                                     NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL);
+        } else {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "continuation stub to unwind the call is absent, hence "
+                        "call will be hung (call-stack id = %"PRIu64")",
+                        frame->root->unique);
         }
 
         mq_reduce_parent_size (this, &oplocal->loc, oplocal->contribution);
@@ -976,7 +1057,7 @@ marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
                 newloc.name++;
         newloc.parent = inode_ref (local->loc.parent);
 
-        mq_rename_update_newpath (this, &newloc);
+        mq_set_inode_xattr (this, &newloc);
 
         loc_wipe (&newloc);
 
@@ -996,7 +1077,7 @@ marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         marker_local_t  *local = NULL, *oplocal = NULL;
         struct gf_flock  lock  = {0, };
@@ -1006,7 +1087,7 @@ marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
 
         if (op_ret < 0) {
                 if (local->err == 0) {
-                        local->err = op_errno;
+                        local->err = op_errno ? op_errno : EINVAL;
                 }
 
                 gf_log (this->name, GF_LOG_WARNING,
@@ -1017,7 +1098,7 @@ marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
         }
 
         if (local->next_lock_on == NULL) {
-                marker_rename_done (frame, NULL, this, 0, 0);
+                marker_rename_done (frame, NULL, this, 0, 0, NULL);
                 goto out;
         }
 
@@ -1031,7 +1112,7 @@ marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
                     marker_rename_done,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock);
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
 
 out:
         return 0;
@@ -1041,7 +1122,7 @@ out:
 int32_t
 marker_rename_release_oldp_lock (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         marker_local_t  *local = NULL, *oplocal = NULL;
         struct gf_flock  lock  = {0, };
@@ -1067,7 +1148,7 @@ marker_rename_release_oldp_lock (call_frame_t *frame, void *cookie,
                     marker_rename_release_newp_lock,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &oplocal->parent_loc, F_SETLKW, &lock);
+                    this->name, &oplocal->parent_loc, F_SETLKW, &lock, NULL);
         return 0;
 }
 
@@ -1076,7 +1157,8 @@ int32_t
 marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
                    struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent)
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
 {
         marker_conf_t  *priv                 = NULL;
         marker_local_t *local                = NULL;
@@ -1111,7 +1193,7 @@ marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 stub = fop_rename_cbk_stub (frame, default_rename_cbk, op_ret,
                                             op_errno, buf, preoldparent,
                                             postoldparent, prenewparent,
-                                            postnewparent);
+                                            postnewparent, xdata);
                 if (stub == NULL) {
                         local->err = ENOMEM;
                         goto quota_err;
@@ -1141,7 +1223,7 @@ marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 STACK_WIND_COOKIE (frame, marker_rename_release_oldp_lock,
                                    frame->cookie, FIRST_CHILD(this),
                                    FIRST_CHILD(this)->fops->removexattr,
-                                   &newloc, contri_key);
+                                   &newloc, contri_key, NULL);
 
                 loc_wipe (&newloc);
         } else {
@@ -1149,7 +1231,7 @@ marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
                                      preoldparent, postoldparent, prenewparent,
-                                     postnewparent);
+                                     postnewparent, xdata);
 
                 if ((op_ret < 0) || (local == NULL)) {
                         goto out;
@@ -1172,15 +1254,14 @@ out:
         return 0;
 
 quota_err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
 
 int32_t
 marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
-
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         marker_local_t       *local           = NULL, *oplocal = NULL;
         char                  contri_key[512] = {0, };
@@ -1195,7 +1276,7 @@ marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
                 MARKER_RESET_UID_GID (frame, frame->root, local);
 
         if ((op_ret < 0) && (op_errno != ENOATTR)) {
-                local->err = op_errno;
+                local->err = op_errno ? op_errno : EINVAL;
                 gf_log (this->name, GF_LOG_WARNING,
                         "fetching contribution values from %s (gfid:%s) "
                         "failed (%s)", local->loc.path,
@@ -1207,7 +1288,7 @@ marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->loc.inode != NULL) {
                 GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
                 if (ret < 0) {
-                        local->err = errno;
+                        local->err = errno ? errno : ENOMEM;
                         goto err;
                 }
 
@@ -1219,12 +1300,12 @@ marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
 
         STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename, &oplocal->loc,
-                    &local->loc);
+                    &local->loc, NULL);
 
         return 0;
 
 err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
@@ -1232,7 +1313,7 @@ err:
 int32_t
 marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *dict)
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         marker_local_t *local           = NULL, *oplocal = NULL;
         char            contri_key[512] = {0, };
@@ -1247,7 +1328,7 @@ marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
                 MARKER_RESET_UID_GID (frame, frame->root, local);
 
         if ((op_ret < 0) && (op_errno != ENOATTR)) {
-                local->err = op_errno;
+                local->err = op_errno ? op_errno : EINVAL;
                 gf_log (this->name, GF_LOG_WARNING,
                         "fetching contribution values from %s (gfid:%s) "
                         "failed (%s)", oplocal->loc.path,
@@ -1258,7 +1339,7 @@ marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
 
         GET_CONTRI_KEY (contri_key, oplocal->loc.parent->gfid, ret);
         if (ret < 0) {
-                local->err = errno;
+                local->err = errno ? errno : ENOMEM;
                 goto err;
         }
 
@@ -1268,7 +1349,7 @@ marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
         if (local->loc.inode != NULL) {
                 GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
                 if (ret < 0) {
-                        local->err = errno;
+                        local->err = errno ? errno : ENOMEM;
                         goto err;
                 }
 
@@ -1284,14 +1365,14 @@ marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
                 STACK_WIND_COOKIE (frame, marker_do_rename,
                                    frame->cookie, FIRST_CHILD(this),
                                    FIRST_CHILD(this)->fops->getxattr,
-                                   &local->loc, contri_key);
+                                   &local->loc, contri_key, NULL);
         } else {
-                marker_do_rename (frame, NULL, this, 0, 0, NULL);
+                marker_do_rename (frame, NULL, this, 0, 0, NULL, NULL);
         }
 
         return 0;
 err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
@@ -1299,7 +1380,7 @@ err:
 int32_t
 marker_get_oldpath_contribution (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         marker_local_t *local           = NULL, *oplocal = NULL;
         char            contri_key[512] = {0, };
@@ -1309,7 +1390,7 @@ marker_get_oldpath_contribution (call_frame_t *frame, void *cookie,
         oplocal = local->oplocal;
 
         if (op_ret < 0) {
-                local->err = op_errno;
+                local->err = op_errno ? op_errno : EINVAL;
                 gf_log (this->name, GF_LOG_WARNING,
                         "cannot hold inodelk on %s (gfid:%s) (%s)",
                         local->next_lock_on->path,
@@ -1320,7 +1401,7 @@ marker_get_oldpath_contribution (call_frame_t *frame, void *cookie,
 
         GET_CONTRI_KEY (contri_key, oplocal->loc.parent->gfid, ret);
         if (ret < 0) {
-                local->err = errno;
+                local->err = errno ? errno : ENOMEM;
                 goto quota_err;
         }
 
@@ -1338,20 +1419,20 @@ marker_get_oldpath_contribution (call_frame_t *frame, void *cookie,
         STACK_WIND_COOKIE (frame, marker_get_newpath_contribution,
                            frame->cookie, FIRST_CHILD(this),
                            FIRST_CHILD(this)->fops->getxattr,
-                           &oplocal->loc, contri_key);
+                           &oplocal->loc, contri_key, NULL);
         return 0;
 
 quota_err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         return 0;
 
 lock_err:
         if ((local->next_lock_on == NULL)
             || (local->next_lock_on == &local->parent_loc)) {
                 local->next_lock_on = NULL;
-                marker_rename_release_oldp_lock (frame, NULL, this, 0, 0);
+                marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
         } else {
-                marker_rename_release_newp_lock (frame, NULL, this, 0, 0);
+                marker_rename_release_newp_lock (frame, NULL, this, 0, 0, NULL);
         }
 
         return 0;
@@ -1360,7 +1441,7 @@ lock_err:
 
 int32_t
 marker_rename_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t  *local = NULL, *oplocal = NULL;
         loc_t           *loc   = NULL;
@@ -1376,7 +1457,7 @@ marker_rename_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         loc = &local->parent_loc;
                 }
 
-                local->err = op_errno;
+                local->err = op_errno ? op_errno : EINVAL;
                 gf_log (this->name, GF_LOG_WARNING,
                         "cannot hold inodelk on %s (gfid:%s) (%s)",
                         loc->path, uuid_utoa (loc->inode->gfid),
@@ -1395,22 +1476,22 @@ marker_rename_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->inodelk,
                             this->name, local->next_lock_on,
-                            F_SETLKW, &lock);
+                            F_SETLKW, &lock, NULL);
         } else {
-                marker_get_oldpath_contribution (frame, 0, this, 0, 0);
+                marker_get_oldpath_contribution (frame, 0, this, 0, 0, NULL);
         }
 
         return 0;
 
 err:
-        marker_rename_done (frame, NULL, this, 0, 0);
+        marker_rename_done (frame, NULL, this, 0, 0, NULL);
         return 0;
 }
 
 
 int32_t
 marker_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
+               loc_t *newloc, dict_t *xdata)
 {
         int32_t         ret              = 0;
         marker_local_t *local            = NULL;
@@ -1479,18 +1560,18 @@ marker_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->inodelk,
                     this->name, lock_on,
-                    F_SETLKW, &lock);
+                    F_SETLKW, &lock, NULL);
 
         return 0;
 
 rename_wind:
         STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
+                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
         return 0;
 err:
         STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL);
+                             NULL, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1499,7 +1580,7 @@ err:
 int32_t
 marker_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1514,7 +1595,7 @@ marker_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1534,7 +1615,8 @@ out:
 }
 
 int32_t
-marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1555,10 +1637,10 @@ marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
                 goto err;
 wind:
         STACK_WIND (frame, marker_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1567,7 +1649,7 @@ err:
 int32_t
 marker_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1582,7 +1664,7 @@ marker_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1601,7 +1683,8 @@ out:
 }
 
 int32_t
-marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1622,10 +1705,10 @@ marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 goto err;
 wind:
         STACK_WIND (frame, marker_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -1635,7 +1718,7 @@ int32_t
 marker_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         marker_conf_t      *priv    = NULL;
         marker_local_t     *local   = NULL;
@@ -1650,11 +1733,14 @@ marker_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
         priv = this->private;
 
         if (priv->feature_enabled & GF_QUOTA)
@@ -1670,7 +1756,7 @@ out:
 
 int
 marker_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                loc_t *loc, dict_t *params)
+                loc_t *loc, mode_t umask, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1691,11 +1777,12 @@ marker_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
                 goto err;
 wind:
         STACK_WIND (frame, marker_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, params);
+                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                    xdata);
         return 0;
 err:
         STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL);
+                             NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1704,7 +1791,7 @@ int32_t
 marker_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1719,11 +1806,14 @@ marker_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent);
+                             buf, preparent, postparent, xdata);
 
         if (op_ret == -1 ||  local == NULL)
                 goto out;
 
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
         priv = this->private;
 
         if ((priv->feature_enabled & GF_QUOTA) && (S_ISREG (local->mode))) {
@@ -1740,7 +1830,7 @@ out:
 
 int
 marker_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, dict_t *parms)
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1763,11 +1853,216 @@ marker_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 goto err;
 wind:
         STACK_WIND (frame, marker_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, parms);
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                    xdata);
         return 0;
 err:
         STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL);
+                             NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_fallocate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "fallocating a file ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (fallocate, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_fallocate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		    xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred during discard",
+                        strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_discard_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (discard, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int32_t
+marker_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred during zerofill",
+                        strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_zerofill_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
+
         return 0;
 }
 
@@ -1827,7 +2122,7 @@ call_from_sp_client_to_reset_tmfile (call_frame_t *frame,
                 op_errno = EINVAL;
         }
 out:
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
 
         return 0;
 }
@@ -1835,7 +2130,7 @@ out:
 
 int32_t
 marker_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1849,7 +2144,7 @@ marker_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1864,16 +2159,150 @@ out:
         return 0;
 }
 
+int
+remove_quota_keys (dict_t *dict, char *k, data_t *v, void *data)
+{
+	call_frame_t 	*frame = data;
+	marker_local_t 	*local = frame->local;
+	xlator_t 	*this = frame->this;
+	int 		ret = -1;
+
+	ret = syncop_removexattr (FIRST_CHILD (this), &local->loc, k, 0);
+	if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove "
+			"extended attribute: %s", local->loc.path, k);
+                return -1;
+	}
+	return 0;
+}
+
+int
+quota_xattr_cleaner_cbk (int ret, call_frame_t *frame, void *args)
+{
+        dict_t *xdata = args;
+        int op_ret = -1;
+        int op_errno = 0;
+	marker_local_t *local = NULL;
+
+	local = frame->local;
+	frame->local = NULL;
+
+        op_ret   = (ret < 0)? -1: 0;
+        op_errno = -ret;
+
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+	marker_local_unref (local);
+        return ret;
+}
+
+int
+quota_xattr_cleaner (void *args)
+{
+        struct synctask *task  = NULL;
+        call_frame_t    *frame = NULL;
+        xlator_t        *this  = NULL;
+        marker_local_t   *local = NULL;
+        dict_t          *xdata = NULL;
+        int             ret    = -1;
+
+        task = synctask_get ();
+        if (!task)
+                goto out;
+
+        frame = task->frame;
+        this  = frame->this;
+        local = frame->local;
+
+        ret = syncop_listxattr (FIRST_CHILD(this), &local->loc, &xdata);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+
+	ret = dict_foreach_fnmatch (xdata, "trusted.glusterfs.quota.*",
+                                    remove_quota_keys, frame);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+	ret = dict_foreach_fnmatch (xdata, PGFID_XATTR_KEY_PREFIX"*",
+                                    remove_quota_keys, frame);
+        if (ret == -1) {
+                ret = -errno;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (xdata)
+                dict_unref (xdata);
+
+        return ret;
+}
+
+int
+marker_do_xattr_cleanup (call_frame_t *frame, xlator_t *this, dict_t *xdata,
+                        loc_t *loc)
+{
+        int           ret       = -1;
+        marker_local_t *local    = NULL;
+
+        local = mem_get0 (this->local_pool);
+	if (!local)
+		goto out;
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        loc_copy (&local->loc, loc);
+        ret = synctask_new (this->ctx->env, quota_xattr_cleaner,
+			    quota_xattr_cleaner_cbk, frame, xdata);
+        if (ret) {
+		gf_log (this->name, GF_LOG_ERROR, "Failed to create synctask "
+			"for cleaning up quota extended attributes");
+                goto out;
+	}
+
+        ret = 0;
+out:
+        if (ret) {
+		frame->local = NULL;
+                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, xdata);
+		marker_local_unref (local);
+        }
+        return ret;
+}
+
+static inline gf_boolean_t
+marker_xattr_cleanup_cmd (dict_t *dict)
+{
+        return (dict_get (dict, VIRTUAL_QUOTA_XATTR_CLEANUP_KEY) != NULL);
+}
+
 int32_t
 marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                 int32_t flags)
+                 int32_t flags, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+        int32_t          ret            = 0;
+        marker_local_t  *local          = NULL;
+        marker_conf_t   *priv           = NULL;
+        int              op_errno       = ENOMEM;
 
         priv = this->private;
 
+        if (marker_xattr_cleanup_cmd (dict)) {
+                if (frame->root->uid != 0 || frame->root->gid != 0) {
+                        op_errno = EPERM;
+                        ret = -1;
+                        goto err;
+                }
+
+                /* The following function does the cleanup and then unwinds the
+                 * corresponding call*/
+                loc_path (loc, NULL);
+                marker_do_xattr_cleanup (frame, this, xdata, loc);
+                return 0;
+        }
+
         if (priv->feature_enabled == 0)
                 goto wind;
 
@@ -1891,10 +2320,10 @@ marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
                 goto err;
 wind:
         STACK_WIND (frame, marker_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags);
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM);
+        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
@@ -1902,7 +2331,7 @@ err:
 
 int32_t
 marker_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1916,7 +2345,7 @@ marker_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -1933,7 +2362,7 @@ out:
 
 int32_t
 marker_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                  int32_t flags)
+                  int32_t flags, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -1958,10 +2387,10 @@ marker_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
                 goto err;
 wind:
         STACK_WIND (frame, marker_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags);
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM);
+        STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
 
         return 0;
 }
@@ -1970,7 +2399,7 @@ err:
 int32_t
 marker_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost)
+                     struct iatt *statpost, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -1985,7 +2414,7 @@ marker_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         frame->local = NULL;
 
         STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -2003,7 +2432,7 @@ out:
 
 int32_t
 marker_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -2024,10 +2453,10 @@ marker_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto err;
 wind:
         STACK_WIND (frame, marker_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
+                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -2036,7 +2465,7 @@ err:
 int32_t
 marker_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                    struct iatt *statpost)
+                    struct iatt *statpost, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -2054,7 +2483,7 @@ marker_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -2071,7 +2500,7 @@ out:
 
 int32_t
 marker_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -2092,10 +2521,10 @@ marker_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto err;
 wind:
         STACK_WIND (frame, marker_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL);
+        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -2103,7 +2532,7 @@ err:
 
 int32_t
 marker_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         marker_local_t     *local   = NULL;
         marker_conf_t      *priv    = NULL;
@@ -2117,7 +2546,7 @@ marker_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
@@ -2134,7 +2563,7 @@ out:
 
 int32_t
 marker_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name)
+                    const char *name, dict_t *xdata)
 {
         int32_t          ret   = 0;
         marker_local_t  *local = NULL;
@@ -2155,10 +2584,10 @@ marker_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto err;
 wind:
         STACK_WIND (frame, marker_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name);
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM);
+        STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
 
         return 0;
 }
@@ -2181,15 +2610,21 @@ marker_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         frame->local = NULL;
 
-        if (!op_ret && local && uuid_is_null (local->loc.gfid))
-                        uuid_copy (local->loc.gfid, inode->gfid);
-
         STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
                              dict, postparent);
 
         if (op_ret == -1 || local == NULL)
                 goto out;
 
+        /* copy the gfid from the stat structure instead of inode,
+         * since if the lookup is fresh lookup, then the inode
+         * would have not yet linked to the inode table which happens
+         * in protocol/server.
+         */
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+
         priv = this->private;
 
         if (priv->feature_enabled & GF_QUOTA) {
@@ -2235,21 +2670,94 @@ err:
         return 0;
 }
 
+
 int
-marker_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, gf_dirent_t *entries)
+marker_build_ancestry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, gf_dirent_t *entries,
+                           dict_t *xdata)
 {
         gf_dirent_t *entry = NULL;
+        loc_t        loc    = {0, };
+        inode_t     *parent = NULL;
+
+        if ((op_ret <= 0) || (entries == NULL)) {
+                goto out;
+        }
+
+
+        list_for_each_entry (entry, &entries->list, list) {
+                if (entry->inode == entry->inode->table->root) {
+                        loc.path = gf_strdup ("/");
+                        inode_unref (parent);
+                        parent = NULL;
+                }
+
+                loc.inode = inode_ref (entry->inode);
+
+                if (parent != NULL) {
+                        loc.parent = inode_ref (parent);
+                        uuid_copy (loc.pargfid, parent->gfid);
+                }
+
+                uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
+
+                mq_xattr_state (this, &loc, entry->dict, entry->d_stat);
+
+                inode_unref (parent);
+                parent = inode_ref (entry->inode);
+                loc_wipe (&loc);
+        }
+
+        if (parent)
+                inode_unref (parent);
+
+out:
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+        return 0;
+}
+
+int
+marker_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
+{
+        gf_dirent_t    *entry = NULL;
+        marker_conf_t  *priv  = NULL;
+        marker_local_t *local = NULL;
+        loc_t           loc   = {0, };
 
         if (op_ret <= 0)
                 goto unwind;
 
+        priv = this->private;
+        local = frame->local;
+
+        if (!(priv->feature_enabled & GF_QUOTA) || (local == NULL)) {
+                goto unwind;
+        }
+
         list_for_each_entry (entry, &entries->list, list) {
-                /* TODO: fill things */
+                if ((strcmp (entry->d_name, ".") == 0) ||
+                    (strcmp (entry->d_name, "..") == 0))
+                        continue;
+
+                loc.inode = inode_ref (entry->inode);
+                loc.parent = inode_ref (local->loc.inode);
+
+                uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
+                uuid_copy (loc.pargfid, loc.parent->gfid);
+
+                mq_xattr_state (this, &loc, entry->dict, entry->d_stat);
+
+                loc_wipe (&loc);
         }
 
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        local = frame->local;
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+        marker_local_unref (local);
 
         return 0;
 }
@@ -2258,20 +2766,36 @@ int
 marker_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                  off_t offset, dict_t *dict)
 {
-        marker_conf_t  *priv    = NULL;
+        marker_conf_t  *priv  = NULL;
+        loc_t           loc   = {0, };
+        marker_local_t *local = NULL;
 
         priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+        if ((dict != NULL) && dict_get (dict, GET_ANCESTRY_DENTRY_KEY)) {
+                STACK_WIND (frame, marker_build_ancestry_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readdirp,
+                            fd, size, offset, dict);
+        } else {
+                if (priv->feature_enabled & GF_QUOTA) {
+                        local = mem_get0 (this->local_pool);
 
-        if ((priv->feature_enabled & GF_QUOTA) && dict)
-                mq_req_xattr (this, NULL, dict);
+                        MARKER_INIT_LOCAL (frame, local);
 
-wind:
-        STACK_WIND (frame, marker_readdirp_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
+                        loc.parent = local->loc.inode = inode_ref (fd->inode);
+
+                        if (dict == NULL)
+                                dict = dict_new ();
+
+                        mq_req_xattr (this, &loc, dict);
+                }
+
+                STACK_WIND (frame, marker_readdirp_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readdirp,
+                            fd, size, offset, dict);
+        }
 
         return 0;
 }
@@ -2377,14 +2901,11 @@ marker_xtime_priv_cleanup (xlator_t *this)
 
         GF_VALIDATE_OR_GOTO (this->name, priv, out);
 
-        if (priv->volume_uuid != NULL)
-                GF_FREE (priv->volume_uuid);
+        GF_FREE (priv->volume_uuid);
 
-        if (priv->timestamp_file != NULL)
-                GF_FREE (priv->timestamp_file);
+        GF_FREE (priv->timestamp_file);
 
-        if (priv->marker_xattr != NULL)
-                GF_FREE (priv->marker_xattr);
+        GF_FREE (priv->marker_xattr);
 out:
         return;
 }
@@ -2412,7 +2933,7 @@ out:
 int32_t
 reconfigure (xlator_t *this, dict_t *options)
 {
-        int32_t         ret     = -1;
+        int32_t         ret     = 0;
         data_t         *data    = NULL;
         gf_boolean_t    flag    = _gf_false;
         marker_conf_t  *priv    = NULL;
@@ -2453,11 +2974,17 @@ reconfigure (xlator_t *this, dict_t *options)
                                         "xtime updation will fail");
                         } else {
                                 priv->feature_enabled |= GF_XTIME;
+                                data = dict_get (options, "gsync-force-xtime");
+                                if (!data)
+                                        goto out;
+                                ret = gf_string2boolean (data->data, &flag);
+                                if (ret == 0 && flag)
+                                        priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
                         }
                 }
         }
 out:
-        return 0;
+        return ret;
 }
 
 
@@ -2513,9 +3040,16 @@ init (xlator_t *this)
                                 goto err;
 
                         priv->feature_enabled |= GF_XTIME;
+                        data = dict_get (options, "gsync-force-xtime");
+                        if (!data)
+                                goto cont;
+                        ret = gf_string2boolean (data->data, &flag);
+                        if (ret == 0 && flag)
+                                priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
                 }
         }
 
+ cont:
         this->local_pool = mem_pool_new (marker_local_t, 128);
         if (!this->local_pool) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -2577,6 +3111,9 @@ struct xlator_fops fops = {
         .removexattr = marker_removexattr,
         .getxattr    = marker_getxattr,
         .readdirp    = marker_readdirp,
+	.fallocate   = marker_fallocate,
+	.discard     = marker_discard,
+        .zerofill    = marker_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -2588,5 +3125,6 @@ struct volume_options options[] = {
         {.key = {"timestamp-file"}},
         {.key = {"quota"}},
         {.key = {"xtime"}},
+        {.key = {"gsync-force-xtime"}},
         {.key = {NULL}}
 };
diff --git a/xlators/features/marker/src/marker.h b/xlators/features/marker/src/marker.h
index 3dcb41f8a..23d1580f0 100644
--- a/xlators/features/marker/src/marker.h
+++ b/xlators/features/marker/src/marker.h
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _MARKER_H
 #define _MARKER_H
 
@@ -37,8 +28,9 @@
 #define TIMESTAMP_FILE      "timestamp-file"
 
 enum {
-        GF_QUOTA=1,
-        GF_XTIME=2
+        GF_QUOTA             = 1,
+        GF_XTIME             = 2,
+        GF_XTIME_GSYNC_FORCE = 4,
 };
 
 /*initialize the local variable*/
@@ -117,6 +109,10 @@ struct marker_local{
 
         quota_inode_ctx_t    *ctx;
         inode_contribution_t *contri;
+
+        int xflag;
+        dict_t *xdata;
+        gf_boolean_t skip_txn;
 };
 typedef struct marker_local marker_local_t;
 
diff --git a/xlators/features/marker/utils/Makefile.am b/xlators/features/marker/utils/Makefile.am
deleted file mode 100644
index 556951d9f..000000000
--- a/xlators/features/marker/utils/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = syncdaemon src
-
-CLEANFILES =
diff --git a/xlators/features/marker/utils/src/Makefile.am b/xlators/features/marker/utils/src/Makefile.am
deleted file mode 100644
index 73c99cb76..000000000
--- a/xlators/features/marker/utils/src/Makefile.am
+++ /dev/null
@@ -1,22 +0,0 @@
-gsyncddir = $(libexecdir)/glusterfs
-
-gsyncd_PROGRAMS = gsyncd
-
-gsyncd_SOURCES = gsyncd.c procdiggy.c
-
-gsyncd_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-gsyncd_LDFLAGS = $(GF_LDFLAGS) $(GF_GLUSTERFS_LDFLAGS)
-
-noinst_HEADERS = procdiggy.h
-
-AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src\
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-	-DSBIN_DIR=\"$(sbindir)\" -DPYTHON=\"$(PYTHON)\"
-
-
-CLEANFILES =
-
-$(top_builddir)/libglusterfs/src/libglusterfs.la:
-	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/xlators/features/marker/utils/src/gsyncd.c b/xlators/features/marker/utils/src/gsyncd.c
deleted file mode 100644
index d554e562b..000000000
--- a/xlators/features/marker/utils/src/gsyncd.c
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-
-#include "common-utils.h"
-#include "run.h"
-#include "procdiggy.h"
-
-#define _GLUSTERD_CALLED_ "_GLUSTERD_CALLED_"
-#define _GSYNCD_DISPATCHED_ "_GSYNCD_DISPATCHED_"
-#define GSYNCD_CONF "geo-replication/gsyncd.conf"
-#define GSYNCD_PY "gsyncd.py"
-#define RSYNC "rsync"
-
-int restricted = 0;
-
-static int
-duplexpand (void **buf, size_t tsiz, size_t *len)
-{
-        size_t osiz = tsiz * *len;
-
-        *buf = realloc (*buf, osiz << 1);
-        if (!buf)
-                return -1;
-
-        memset ((char *)*buf + osiz, 0, osiz);
-        *len <<= 1;
-
-        return 0;
-}
-
-static int
-str2argv (char *str, char ***argv)
-{
-        char *p         = NULL;
-        int argc        = 0;
-        size_t argv_len = 32;
-        int ret         = 0;
-
-        assert (str);
-        str = strdup (str);
-        if (!str)
-                return -1;
-
-        *argv = calloc (argv_len, sizeof (**argv));
-        if (!*argv)
-                goto error;
-
-        while ((p = strtok (str, " "))) {
-                str = NULL;
-
-                argc++;
-                if (argc == argv_len) {
-                        ret = duplexpand ((void *)argv,
-                                          sizeof (**argv),
-                                          &argv_len);
-                        if (ret == -1)
-                                goto error;
-                }
-                (*argv)[argc - 1] = p;
-        }
-
-        return argc;
-
- error:
-        fprintf (stderr, "out of memory\n");
-        return -1;
-}
-
-static int
-invoke_gsyncd (int argc, char **argv)
-{
-        char config_file[PATH_MAX] = {0,};
-        size_t gluster_workdir_len = 0;
-        runner_t runner            = {0,};
-        int i                      = 0;
-        int j                      = 0;
-        char *nargv[argc + 4];
-
-        if (restricted) {
-                /* in restricted mode we forcibly use the system-wide config */
-                runinit (&runner);
-                runner_add_args (&runner, SBIN_DIR"/gluster",
-                                 "--log-file=-", "system::", "getwd",
-                                 NULL);
-                runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-                if (runner_start (&runner) == 0 &&
-                    fgets (config_file, PATH_MAX,
-                           runner_chio (&runner, STDOUT_FILENO)) != NULL &&
-                    config_file[strlen (config_file) - 1] == '\n' &&
-                    runner_end (&runner) == 0)
-                        gluster_workdir_len = strlen (config_file) - 1;
-
-                if (gluster_workdir_len) {
-                        if (gluster_workdir_len + 1 + strlen (GSYNCD_CONF) + 1 >
-                            PATH_MAX)
-                                goto error;
-                        config_file[gluster_workdir_len] = '/';
-                        strcat (config_file, GSYNCD_CONF);
-                } else
-                        goto error;
-
-                if (setenv ("_GSYNCD_RESTRICTED_", "1", 1) == -1)
-                        goto error;
-        }
-
-        if (chdir ("/") == -1)
-                goto error;
-
-        j = 0;
-        nargv[j++] = PYTHON;
-        nargv[j++] = GSYNCD_PREFIX"/python/syncdaemon/"GSYNCD_PY;
-        for (i = 1; i < argc; i++)
-                nargv[j++] = argv[i];
-        if (config_file[0]) {
-                nargv[j++] = "-c";
-                nargv[j++] = config_file;
-        }
-        nargv[j++] = NULL;
-
-        execvp (PYTHON, nargv);
-
-        fprintf (stderr, "exec of "PYTHON" failed\n");
-        return 127;
-
- error:
-        fprintf (stderr, "gsyncd initializaion failed\n");
-        return 1;
-}
-
-
-static int
-find_gsyncd (pid_t pid, pid_t ppid, char *name, void *data)
-{
-        char buf[NAME_MAX * 2] = {0,};
-        char *p                = NULL;
-        int zeros              = 0;
-        int ret                = 0;
-        int fd                 = -1;
-        pid_t *pida            = (pid_t *)data;
-
-        if (ppid != pida[0])
-                return 0;
-
-        ret = gf_asprintf (&p, PROC"/%d/cmdline", pid);
-        if (ret == -1) {
-                fprintf (stderr, "out of memory\n");
-                return -1;
-        }
-
-        fd = open (p, O_RDONLY);
-        if (fd == -1)
-                return 0;
-        ret = read (fd, buf, sizeof (buf));
-        close (fd);
-        if (ret == -1)
-                return 0;
-        for (zeros = 0, p = buf; zeros < 2 && p < buf + ret; p++)
-                zeros += !*p;
-
-        ret = 0;
-        switch (zeros) {
-        case 2:
-                if ((strcmp (basename (buf), basename (PYTHON)) ||
-                     strcmp (basename (buf + strlen (buf) + 1), GSYNCD_PY)) == 0) {
-                        ret = 1;
-                        break;
-                }
-                /* fallthrough */
-        case 1:
-                if (strcmp (basename (buf), GSYNCD_PY) == 0)
-                        ret = 1;
-        }
-
-        if (ret == 1) {
-                if (pida[1] != -1) {
-                        fprintf (stderr, GSYNCD_PY" sibling is not unique");
-                        return -1;
-                }
-                pida[1] = pid;
-        }
-
-        return 0;
-}
-
-static int
-invoke_rsync (int argc, char **argv)
-{
-        int i                  = 0;
-        char *p                = NULL;
-        pid_t pid              = -1;
-        pid_t ppid             = -1;
-        pid_t pida[]           = {-1, -1};
-        char *name             = NULL;
-        char buf[PATH_MAX + 1] = {0,};
-        int ret                = 0;
-
-        assert (argv[argc] == NULL);
-
-        if (argc < 2 || strcmp (argv[1], "--server") != 0)
-                goto error;
-
-        for (i = 2; i < argc && argv[i][0] == '-'; i++);
-
-        if (!(i == argc - 2 && strcmp (argv[i], ".") == 0 && argv[i + 1][0] == '/')) {
-                fprintf (stderr, "need an rsync invocation without protected args\n");
-                goto error;
-        }
-
-        /* look up sshd we are spawned from */
-        for (pid = getpid () ;; pid = ppid) {
-                ppid = pidinfo (pid, &name);
-                if (ppid < 0) {
-                        fprintf (stderr, "sshd ancestor not found\n");
-                        goto error;
-                }
-                if (strcmp (name, "sshd") == 0)
-                        break;
-        }
-        /* look up "ssh-sibling" gsyncd */
-        pida[0] = pid;
-        ret = prociter (find_gsyncd, pida);
-        if (ret == -1 || pida[1] == -1) {
-                fprintf (stderr, "gsyncd sibling not found\n");
-                goto error;
-        }
-        /* check if rsync target matches gsyncd target */
-        if (gf_asprintf (&p, PROC"/%d/cwd", pida[1]) == -1) {
-                fprintf (stderr, "out of memory\n");
-                goto error;
-        }
-        ret = readlink (p, buf, sizeof (buf));
-        if (ret == -1 || ret == sizeof (buf))
-                goto error;
-        if (strcmp (argv[argc - 1], "/") == 0 /* root dir cannot be a target */ ||
-            (strcmp (argv[argc - 1], p) /* match against gluster target */ &&
-             strcmp (argv[argc - 1], buf) /* match against file target */) != 0) {
-                fprintf (stderr, "rsync target does not match "GEOREP" session\n");
-                goto error;
-        }
-
-        argv[0] = RSYNC;
-
-        execvp (RSYNC, argv);
-
-        fprintf (stderr, "exec of "RSYNC" failed\n");
-        return 127;
-
- error:
-        fprintf (stderr, "disallowed "RSYNC" invocation\n");
-        return 1;
-}
-
-
-struct invocable {
-        char *name;
-        int (*invoker) (int argc, char **argv);
-};
-
-struct invocable invocables[] = {
-        { "rsync",  invoke_rsync  },
-        { "gsyncd", invoke_gsyncd },
-        { NULL, NULL}
-};
-
-int
-main (int argc, char **argv)
-{
-        char *evas          = NULL;
-        struct invocable *i = NULL;
-        char *b             = NULL;
-        char *sargv         = NULL;
-
-        evas = getenv (_GLUSTERD_CALLED_);
-        if (evas && strcmp (evas, "1") == 0)
-                /* OK, we know glusterd called us, no need to look for further config
-                 * ... altough this conclusion should not inherit to our children
-                 */
-                unsetenv (_GLUSTERD_CALLED_);
-        else {
-                /* we regard all gsyncd invocations unsafe
-                 * that do not come from glusterd and
-                 * therefore restrict it
-                 */
-                restricted = 1;
-
-                if (!getenv (_GSYNCD_DISPATCHED_)) {
-                        evas = getenv ("SSH_ORIGINAL_COMMAND");
-                        if (evas)
-                                sargv = evas;
-                        else {
-                                evas = getenv ("SHELL");
-                                if (evas && strcmp (basename (evas), "gsyncd") == 0 &&
-                                    argc == 3 && strcmp (argv[1], "-c") == 0)
-                                        sargv = argv[2];
-                        }
-                }
-
-        }
-
-        if (!(sargv && restricted))
-                return invoke_gsyncd (argc, argv);
-
-        argc = str2argv (sargv, &argv);
-        if (argc == -1 || setenv (_GSYNCD_DISPATCHED_, "1", 1) == -1) {
-                fprintf (stderr, "internal error\n");
-                return 1;
-        }
-
-        b = basename (argv[0]);
-        for (i = invocables; i->name; i++) {
-                if (strcmp (b, i->name) == 0)
-                        return i->invoker (argc, argv);
-        }
-
-        fprintf (stderr, "invoking %s in restricted SSH session is not allowed\n",
-                 b);
-
-        return 1;
-}
diff --git a/xlators/features/marker/utils/src/procdiggy.c b/xlators/features/marker/utils/src/procdiggy.c
deleted file mode 100644
index fc0f97999..000000000
--- a/xlators/features/marker/utils/src/procdiggy.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-#include <ctype.h>
-
-#include "common-utils.h"
-#include "procdiggy.h"
-
-pid_t
-pidinfo (pid_t pid, char **name)
-{
-        char buf[NAME_MAX * 2] = {0,};
-        FILE *f                = NULL;
-        char *p                = NULL;
-        int ret                = 0;
-
-        ret = gf_asprintf (&p, PROC"/%d/status", pid);
-        if (ret == -1)
-                goto oom;
-
-        f = fopen (p, "r");
-        if (!f)
-                return -1;
-
-        if (name)
-                *name = NULL;
-        for (;;) {
-                memset (buf, 0, sizeof (buf));
-                if (fgets (buf, sizeof (buf), f) == NULL ||
-                    buf[strlen (buf) - 1] != '\n') {
-                        pid = -1;
-                        goto out;
-                }
-                buf[strlen (buf) -1] = '\0';
-
-                if (name && !*name) {
-                        p = strtail (buf, "Name:");
-                        if (p) {
-                                while (isspace (*++p));
-                                *name = gf_strdup (p);
-                                if (!*name)
-                                        goto oom;
-                                continue;
-                        }
-                }
-
-                p = strtail (buf, "PPid:");
-                if (p)
-                        break;
-        }
-
-        while (isspace (*++p));
-        ret = gf_string2int (p, &pid);
-        if (ret == -1)
-                pid = -1;
-
- out:
-        fclose (f);
-        return pid;
-
- oom:
-        fclose (f);
-        fprintf (stderr, "out of memory\n");
-        return -2;
-}
-
-int
-prociter (int (*proch) (pid_t pid, pid_t ppid, char *name, void *data),
-          void *data)
-{
-        char *name        = NULL;
-        DIR *d            = NULL;
-        struct dirent *de = NULL;
-        pid_t pid         = -1;
-        pid_t ppid        = -1;
-        int ret           = 0;
-
-        d = opendir (PROC);
-        while (errno = 0, de = readdir (d)) {
-                if (gf_string2int (de->d_name, &pid) != -1 && pid >= 0) {
-                        ppid = pidinfo (pid, &name);
-                        switch (ppid) {
-                        case -1: continue;
-                        case -2: return -1;
-                        }
-                        ret = proch (pid, ppid, name, data);
-                        if (ret)
-                                return ret;
-                }
-        }
-        if (errno) {
-                fprintf (stderr, "failed to traverse "PROC" (%s)\n",
-                         strerror (errno));
-                return -1;
-        }
-
-        return 0;
-}
diff --git a/xlators/features/marker/utils/src/procdiggy.h b/xlators/features/marker/utils/src/procdiggy.h
deleted file mode 100644
index f4586de6c..000000000
--- a/xlators/features/marker/utils/src/procdiggy.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#define PROC "/proc"
-
-pid_t pidinfo (pid_t pid, char **name);
-
-int prociter (int (*proch) (pid_t pid, pid_t ppid, char *name, void *data),
-              void *data);
-
diff --git a/xlators/features/marker/utils/syncdaemon/Makefile.am b/xlators/features/marker/utils/syncdaemon/Makefile.am
deleted file mode 100644
index cc7cee102..000000000
--- a/xlators/features/marker/utils/syncdaemon/Makefile.am
+++ /dev/null
@@ -1,6 +0,0 @@
-syncdaemondir = $(libexecdir)/glusterfs/python/syncdaemon
-
-syncdaemon_PYTHON = gconf.py gsyncd.py __init__.py master.py README.md repce.py resource.py configinterface.py syncdutils.py monitor.py libcxattr.py \
-                    $(top_builddir)/contrib/ipaddr-py/ipaddr.py
-
-CLEANFILES =
diff --git a/xlators/features/marker/utils/syncdaemon/README.md b/xlators/features/marker/utils/syncdaemon/README.md
deleted file mode 100644
index d45006932..000000000
--- a/xlators/features/marker/utils/syncdaemon/README.md
+++ /dev/null
@@ -1,81 +0,0 @@
-gsycnd, the Gluster Syncdaemon
-==============================
-
-REQUIREMENTS
-------------
-
-_gsyncd_ is a program which can operate either in _master_ or in _slave_ mode.
-Requirements are categorized according to this.
-
-* supported OS is GNU/Linux
-* Python >= 2.5, or 2.4 with Ctypes (see below) (both)
-* OpenSSH >= 4.0 (master) / SSH2 compliant sshd (eg. openssh) (slave)
-* rsync (both)
-* glusterfs with marker support (master); glusterfs (optional on slave)
-* FUSE; for supported versions consult glusterfs
-
-INSTALLATION
-------------
-
-As of now, the supported way of operation is running from the source directory.
-
-If you use Python 2.4.x, you need to install the [Ctypes module](http://python.net/crew/theller/ctypes/).
-
-CONFIGURATION
--------------
-
-gsyncd tunables are a subset of the long command-line options; for listing them,
-type
-
-    gsyncd.py --help
-
-and see the long options up to "--config-file". (The leading double dash should be omitted;
-interim underscores and dashes are interchangeable.) The set of options bear some resemblance
-to those of glusterfs and rsync.
-
-The config file format matches the following syntax:
-
-      <option1>: <value1>
-      <option2>: <value2>
-      # comment
-
-By default (unless specified by the option `-c`), gsyncd looks for config file at _conf/gsyncd.conf_
-in the source tree.
-
-USAGE
------
-
-gsyncd is a utilitly for continous mirroring, ie. it mirrors master to slave incrementally.
-Assume we have a gluster volume _pop_ at localhost. We try to set up the following mirrors
-for it with gysncd:
-
-1. _/data/mirror_
-2. local gluster volume _yow_
-3. _/data/far_mirror_ at example.com
-4. gluster volume _moz_ at example.com
-
-The respective gsyncd invocations are (demoing some syntax sugaring):
-
-1.
-
-      gsyncd.py gluster://localhost:pop file:///data/mirror
-
-  or short form
-
-      gsyncd.py :pop /data/mirror
-
-2. `gsyncd :pop :yow`
-3.
-
-       gsyncd.py :pop ssh://example.com:/data/far_mirror
-
-  or short form
-
-       gsyncd.py :pop example.com:/data/far_mirror
-
-4. `gsyncd.py :pop example.com::moz`
-
-gsyncd has to be available on both sides; it's location on the remote side has to be specified
-via the "--remote-gsyncd" option (or "remote-gsyncd" config file parameter). (This option can also be
-used for setting options on the remote side, although the suggested mode of operation is to
-set parameters like log file / pid file in the configuration file.)
diff --git a/xlators/features/marker/utils/syncdaemon/__codecheck.py b/xlators/features/marker/utils/syncdaemon/__codecheck.py
deleted file mode 100644
index e3386afba..000000000
--- a/xlators/features/marker/utils/syncdaemon/__codecheck.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os
-import os.path
-import sys
-import tempfile
-import shutil
-
-ipd = tempfile.mkdtemp(prefix = 'codecheck-aux')
-
-try:
-    # add a fake ipaddr module, we don't want to
-    # deal with the real one (just test our code)
-    f = open(os.path.join(ipd, 'ipaddr.py'), 'w')
-    f.write("""
-class IPAddress(object):
-    pass
-class IPNetwork(list):
-    pass
-""")
-    f.close()
-    sys.path.append(ipd)
-
-    fl = os.listdir(os.path.dirname(sys.argv[0]) or '.')
-    fl.sort()
-    for f in fl:
-        if f[-3:] != '.py' or f[0] == '_':
-            continue
-        m = f[:-3]
-        sys.stdout.write('importing %s ...' %  m)
-        __import__(m)
-        print(' OK.')
-
-    def sys_argv_set(a):
-        sys.argv = sys.argv[:1] + a
-
-    gsyncd = sys.modules['gsyncd']
-    for a in [['--help'], ['--version'], ['--canonicalize-escape-url', '/foo']]:
-        print('>>> invoking program with args: %s' % ' '.join(a))
-        pid = os.fork()
-        if not pid:
-            sys_argv_set(a)
-            gsyncd.main()
-        _, r = os.waitpid(pid, 0)
-        if r:
-            raise RuntimeError('invocation failed')
-finally:
-    shutil.rmtree(ipd)
diff --git a/xlators/features/marker/utils/syncdaemon/__init__.py b/xlators/features/marker/utils/syncdaemon/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/xlators/features/marker/utils/syncdaemon/__init__.py
+++ /dev/null
diff --git a/xlators/features/marker/utils/syncdaemon/configinterface.py b/xlators/features/marker/utils/syncdaemon/configinterface.py
deleted file mode 100644
index e55bec519..000000000
--- a/xlators/features/marker/utils/syncdaemon/configinterface.py
+++ /dev/null
@@ -1,224 +0,0 @@
-try:
-    import ConfigParser
-except ImportError:
-    # py 3
-    import configparser as ConfigParser
-import re
-from string import Template
-
-from syncdutils import escape, unescape, norm, update_file, GsyncdError
-
-SECT_ORD = '__section_order__'
-SECT_META = '__meta__'
-config_version = 2.0
-
-re_type = type(re.compile(''))
-
-
-class MultiDict(object):
-    """a virtual dict-like class which functions as the union of underlying dicts"""
-
-    def __init__(self, *dd):
-        self.dicts = dd
-
-    def __getitem__(self, key):
-        val = None
-        for d in self.dicts:
-            if d.get(key):
-                val = d[key]
-        if not val:
-            raise KeyError(key)
-        return val
-
-
-class GConffile(object):
-    """A high-level interface to ConfigParser which flattens the two-tiered
-       config layout by implenting automatic section dispatch based on initial
-       parameters.
-
-    Also ensure section ordering in terms of their time of addition -- a compat
-    hack for Python < 2.7.
-    """
-
-    def _normconfig(self):
-        """normalize config keys by s/-/_/g"""
-        for n, s in self.config._sections.items():
-            if n.find('__') == 0:
-                continue
-            s2 = type(s)()
-            for k, v in s.items():
-                if k.find('__') != 0:
-                    k = norm(k)
-                s2[k] = v
-            self.config._sections[n] = s2
-
-    def __init__(self, path, peers, *dd):
-        """
-        - .path: location of config file
-        - .config: underlying ConfigParser instance
-        - .peers: on behalf of whom we flatten .config
-          (master, or master-slave url pair)
-        - .auxdicts: template subtituents
-        """
-        self.peers = peers
-        self.path = path
-        self.auxdicts = dd
-        self.config = ConfigParser.RawConfigParser()
-        self.config.read(path)
-        self._normconfig()
-
-    def section(self, rx=False):
-        """get the section name of the section representing .peers in .config"""
-        peers = self.peers
-        if not peers:
-            peers = ['.', '.']
-            rx = True
-        if rx:
-            st = 'peersrx'
-        else:
-            st = 'peers'
-        return ' '.join([st] + [escape(u) for u in peers])
-
-    @staticmethod
-    def parse_section(section):
-        """retrieve peers sequence encoded by section name
-           (as urls or regexen, depending on section type)
-        """
-        sl = section.split()
-        st = sl.pop(0)
-        sl = [unescape(u) for u in sl]
-        if st == 'peersrx':
-            sl = [re.compile(u) for u in sl]
-        return sl
-
-    def ord_sections(self):
-        """Return an ordered list of sections.
-
-        Ordering happens based on the auxiliary
-        SECT_ORD section storing indices for each
-        section added through the config API.
-
-        To not to go corrupt in case of manually
-        written config files, we take care to append
-        also those sections which are not registered
-        in SECT_ORD.
-
-        Needed for python 2.{4,5,6} where ConfigParser
-        cannot yet order sections/options internally.
-        """
-        so = {}
-        if self.config.has_section(SECT_ORD):
-            so = self.config._sections[SECT_ORD]
-        so2 = {}
-        for k, v in so.items():
-            if k != '__name__':
-                so2[k] = int(v)
-        tv = 0
-        if so2:
-            tv = max(so2.values()) + 1
-        ss = [s for s in self.config.sections() if s.find('__') != 0]
-        for s in ss:
-            if s in so.keys():
-                continue
-            so2[s] = tv
-            tv += 1
-        def scmp(x, y):
-            return cmp(*(so2[s] for s in (x, y)))
-        ss.sort(scmp)
-        return ss
-
-    def update_to(self, dct, allow_unresolved=False):
-        """update @dct from key/values of ours.
-
-        key/values are collected from .config by filtering the regexp sections
-        according to match, and from .section. The values are treated as templates,
-        which are substituted from .auxdicts and (in case of regexp sections)
-        match groups.
-        """
-        if not self.peers:
-            raise GsyncdError('no peers given, cannot select matching options')
-        def update_from_sect(sect, mud):
-            for k, v in self.config._sections[sect].items():
-                if k == '__name__':
-                    continue
-                if allow_unresolved:
-                    dct[k] = Template(v).safe_substitute(mud)
-                else:
-                    dct[k] = Template(v).substitute(mud)
-        for sect in self.ord_sections():
-            sp = self.parse_section(sect)
-            if isinstance(sp[0], re_type) and len(sp) == len(self.peers):
-                match = True
-                mad = {}
-                for i in range(len(sp)):
-                    m = sp[i].search(self.peers[i])
-                    if not m:
-                        match = False
-                        break
-                    for j in range(len(m.groups())):
-                        mad['match%d_%d' % (i+1, j+1)] = m.groups()[j]
-                if match:
-                    update_from_sect(sect, MultiDict(dct, mad, *self.auxdicts))
-        if self.config.has_section(self.section()):
-            update_from_sect(self.section(), MultiDict(dct, *self.auxdicts))
-
-    def get(self, opt=None):
-        """print the matching key/value pairs from .config,
-           or if @opt given, the value for @opt (according to the
-           logic described in .update_to)
-        """
-        d = {}
-        self.update_to(d, allow_unresolved = True)
-        if opt:
-            opt = norm(opt)
-            v = d.get(opt)
-            if v:
-                print(v)
-        else:
-            for k, v in d.iteritems():
-                if k == '__name__':
-                    continue
-                print("%s: %s" % (k, v))
-
-    def write(self, trfn, opt, *a, **kw):
-        """update on-disk config transactionally
-
-        @trfn is the transaction function
-        """
-        def mergeconf(f):
-            self.config = ConfigParser.RawConfigParser()
-            self.config.readfp(f)
-            self._normconfig()
-            if not self.config.has_section(SECT_META):
-                self.config.add_section(SECT_META)
-            self.config.set(SECT_META, 'version', config_version)
-            return trfn(norm(opt), *a, **kw)
-        def updateconf(f):
-            self.config.write(f)
-        update_file(self.path, updateconf, mergeconf)
-
-    def _set(self, opt, val, rx=False):
-        """set @opt to @val in .section"""
-        sect = self.section(rx)
-        if not self.config.has_section(sect):
-            self.config.add_section(sect)
-            # regarding SECT_ORD, cf. ord_sections
-            if not self.config.has_section(SECT_ORD):
-                self.config.add_section(SECT_ORD)
-            self.config.set(SECT_ORD, sect, len(self.config._sections[SECT_ORD]))
-        self.config.set(sect, opt, val)
-        return True
-
-    def set(self, opt, *a, **kw):
-        """perform ._set transactionally"""
-        self.write(self._set, opt, *a, **kw)
-
-    def _delete(self, opt, rx=False):
-        """delete @opt from .section"""
-        sect = self.section(rx)
-        if self.config.has_section(sect):
-            return self.config.remove_option(sect, opt)
-
-    def delete(self, opt, *a, **kw):
-        """perform ._delete transactionally"""
-        self.write(self._delete, opt, *a, **kw)
diff --git a/xlators/features/marker/utils/syncdaemon/gconf.py b/xlators/features/marker/utils/syncdaemon/gconf.py
deleted file mode 100644
index 146c72a18..000000000
--- a/xlators/features/marker/utils/syncdaemon/gconf.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import os
-
-class GConf(object):
-    """singleton class to store globals
-       shared between gsyncd modules"""
-
-    ssh_ctl_dir = None
-    ssh_ctl_args = None
-    cpid = None
-    pid_file_owned = False
-    log_exit = False
-    permanent_handles = []
-    log_metadata = {}
-
-    @classmethod
-    def setup_ssh_ctl(cls, ctld):
-        cls.ssh_ctl_dir = ctld
-        cls.ssh_ctl_args = ["-oControlMaster=auto", "-S", os.path.join(ctld, "gsycnd-ssh-%r@%h:%p")]
-
-gconf = GConf()
diff --git a/xlators/features/marker/utils/syncdaemon/gsyncd.py b/xlators/features/marker/utils/syncdaemon/gsyncd.py
deleted file mode 100644
index 343bb8cb0..000000000
--- a/xlators/features/marker/utils/syncdaemon/gsyncd.py
+++ /dev/null
@@ -1,370 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import os.path
-import sys
-import time
-import logging
-import signal
-import optparse
-import fcntl
-import fnmatch
-from optparse import OptionParser, SUPPRESS_HELP
-from logging import Logger
-from errno import EEXIST, ENOENT
-
-from ipaddr import IPAddress, IPNetwork
-
-from gconf import gconf
-from syncdutils import FreeObject, norm, grabpidfile, finalize, log_raise_exception
-from syncdutils import GsyncdError, select
-from configinterface import GConffile
-import resource
-from monitor import monitor
-
-class GLogger(Logger):
-    """Logger customizations for gsyncd.
-
-    It implements a log format similar to that of glusterfs.
-    """
-
-    def makeRecord(self, name, level, *a):
-        rv = Logger.makeRecord(self, name, level, *a)
-        rv.nsecs = (rv.created - int(rv.created)) * 1000000
-        fr = sys._getframe(4)
-        callee = fr.f_locals.get('self')
-        if callee:
-            ctx = str(type(callee)).split("'")[1].split('.')[-1]
-        else:
-            ctx = '<top>'
-        if not hasattr(rv, 'funcName'):
-            rv.funcName = fr.f_code.co_name
-        rv.lvlnam = logging.getLevelName(level)[0]
-        rv.ctx = ctx
-        return rv
-
-    @classmethod
-    def setup(cls, **kw):
-        lbl = kw.get('label', "")
-        if lbl:
-            lbl = '(' + lbl + ')'
-        lprm = {'datefmt': "%Y-%m-%d %H:%M:%S",
-                'format': "[%(asctime)s.%(nsecs)d] %(lvlnam)s [%(module)s" + lbl + ":%(lineno)s:%(funcName)s] %(ctx)s: %(message)s"}
-        lprm.update(kw)
-        lvl = kw.get('level', logging.INFO)
-        lprm['level'] = lvl
-        logging.root = cls("root", lvl)
-        logging.setLoggerClass(cls)
-        logging.getLogger().handlers = []
-        logging.basicConfig(**lprm)
-
-def startup(**kw):
-    """set up logging, pidfile grabbing, daemonization"""
-    if getattr(gconf, 'pid_file', None) and kw.get('go_daemon') != 'postconn':
-        if not grabpidfile():
-            sys.stderr.write("pidfile is taken, exiting.\n")
-            sys.exit(2)
-        gconf.pid_file_owned = True
-
-    if kw.get('go_daemon') == 'should':
-        x, y = os.pipe()
-        gconf.cpid = os.fork()
-        if gconf.cpid:
-            os.close(x)
-            sys.exit()
-        os.close(y)
-        os.setsid()
-        dn = os.open(os.devnull, os.O_RDWR)
-        for f in (sys.stdin, sys.stdout, sys.stderr):
-            os.dup2(dn, f.fileno())
-        if getattr(gconf, 'pid_file', None):
-            if not grabpidfile(gconf.pid_file + '.tmp'):
-                raise GsyncdError("cannot grab temporary pidfile")
-            os.rename(gconf.pid_file + '.tmp', gconf.pid_file)
-        # wait for parent to terminate
-        # so we can start up with
-        # no messing from the dirty
-        # ol' bustard
-        select((x,), (), ())
-        os.close(x)
-
-    lkw = {}
-    if gconf.log_level:
-        lkw['level'] = gconf.log_level
-    if kw.get('log_file'):
-        if kw['log_file'] in ('-', '/dev/stderr'):
-            lkw['stream'] = sys.stderr
-        elif kw['log_file'] == '/dev/stdout':
-            lkw['stream'] = sys.stdout
-        else:
-            lkw['filename'] = kw['log_file']
-
-    GLogger.setup(label=kw.get('label'), **lkw)
-
-    lkw.update({'saved_label': kw.get('label')})
-    gconf.log_metadata = lkw
-    gconf.log_exit = True
-
-def main():
-    """main routine, signal/exception handling boilerplates"""
-    signal.signal(signal.SIGTERM, lambda *a: finalize(*a, **{'exval': 1}))
-    GLogger.setup()
-    excont = FreeObject(exval = 0)
-    try:
-        try:
-            main_i()
-        except:
-            log_raise_exception(excont)
-    finally:
-        finalize(exval = excont.exval)
-
-def main_i():
-    """internal main routine
-
-    parse command line, decide what action will be taken;
-    we can either:
-    - query/manipulate configuration
-    - format gsyncd urls using gsyncd's url parsing engine
-    - start service in following modes, in given stages:
-      - monitor: startup(), monitor()
-      - master: startup(), connect_remote(), connect(), service_loop()
-      - slave: startup(), connect(), service_loop()
-    """
-    rconf = {'go_daemon': 'should'}
-
-    def store_abs(opt, optstr, val, parser):
-        if val and val != '-':
-            val = os.path.abspath(val)
-        setattr(parser.values, opt.dest, val)
-    def store_local(opt, optstr, val, parser):
-        rconf[opt.dest] = val
-    def store_local_curry(val):
-        return lambda o, oo, vx, p: store_local(o, oo, val, p)
-    def store_local_obj(op, dmake):
-        return lambda o, oo, vx, p: store_local(o, oo, FreeObject(op=op, **dmake(vx)), p)
-
-    op = OptionParser(usage="%prog [options...] <master> <slave>", version="%prog 0.0.1")
-    op.add_option('--gluster-command-dir', metavar='DIR',   default='')
-    op.add_option('--gluster-log-file',    metavar='LOGF',  default=os.devnull, type=str, action='callback', callback=store_abs)
-    op.add_option('--gluster-log-level',   metavar='LVL')
-    op.add_option('--gluster-params',      metavar='PRMS',  default='')
-    op.add_option('--gluster-cli-options', metavar='OPTS',  default='--log-file=-')
-    op.add_option('--mountbroker',         metavar='LABEL')
-    op.add_option('-p', '--pid-file',      metavar='PIDF',  type=str, action='callback', callback=store_abs)
-    op.add_option('-l', '--log-file',      metavar='LOGF',  type=str, action='callback', callback=store_abs)
-    op.add_option('--state-file',          metavar='STATF', type=str, action='callback', callback=store_abs)
-    op.add_option('--ignore-deletes',      default=False, action='store_true')
-    op.add_option('-L', '--log-level',     metavar='LVL')
-    op.add_option('-r', '--remote-gsyncd', metavar='CMD',   default=os.path.abspath(sys.argv[0]))
-    op.add_option('--volume-id',           metavar='UUID')
-    op.add_option('--session-owner',       metavar='ID')
-    op.add_option('-s', '--ssh-command',   metavar='CMD',   default='ssh')
-    op.add_option('--rsync-command',       metavar='CMD',   default='rsync')
-    op.add_option('--rsync-extra',         metavar='ARGS',  default='-S', help=SUPPRESS_HELP)
-    op.add_option('--timeout',             metavar='SEC',   type=int, default=120)
-    op.add_option('--connection-timeout',  metavar='SEC',   type=int, default=60, help=SUPPRESS_HELP)
-    op.add_option('--sync-jobs',           metavar='N',     type=int, default=3)
-    op.add_option('--turns',               metavar='N',     type=int, default=0, help=SUPPRESS_HELP)
-    op.add_option('--allow-network',       metavar='IPS',   default='')
-
-    op.add_option('-c', '--config-file',   metavar='CONF',  type=str, action='callback', callback=store_local)
-    # duh. need to specify dest or value will be mapped to None :S
-    op.add_option('--monitor', dest='monitor', action='callback', callback=store_local_curry(True))
-    op.add_option('--feedback-fd', dest='feedback_fd', type=int, help=SUPPRESS_HELP, action='callback', callback=store_local)
-    op.add_option('--listen', dest='listen', help=SUPPRESS_HELP,      action='callback', callback=store_local_curry(True))
-    op.add_option('-N', '--no-daemon', dest="go_daemon",    action='callback', callback=store_local_curry('dont'))
-    op.add_option('--debug', dest="go_daemon",              action='callback', callback=lambda *a: (store_local_curry('dont')(*a),
-                                                                                                    setattr(a[-1].values, 'log_file', '-'),
-                                                                                                    setattr(a[-1].values, 'log_level', 'DEBUG'))),
-
-    for a in ('check', 'get'):
-        op.add_option('--config-' + a,      metavar='OPT',  type=str, dest='config', action='callback',
-                      callback=store_local_obj(a, lambda vx: {'opt': vx}))
-    op.add_option('--config-get-all', dest='config', action='callback', callback=store_local_obj('get', lambda vx: {'opt': None}))
-    for m in ('', '-rx', '-glob'):
-        # call this code 'Pythonic' eh?
-        # have to define a one-shot local function to be able to inject (a value depending on the)
-        # iteration variable into the inner lambda
-        def conf_mod_opt_regex_variant(rx):
-            op.add_option('--config-set' + m,   metavar='OPT VAL', type=str, nargs=2, dest='config', action='callback',
-                          callback=store_local_obj('set', lambda vx: {'opt': vx[0], 'val': vx[1], 'rx': rx}))
-            op.add_option('--config-del' + m,   metavar='OPT',  type=str, dest='config', action='callback',
-                          callback=store_local_obj('del', lambda vx: {'opt': vx, 'rx': rx}))
-        conf_mod_opt_regex_variant(m and m[1:] or False)
-
-    op.add_option('--normalize-url',           dest='url_print', action='callback', callback=store_local_curry('normal'))
-    op.add_option('--canonicalize-url',        dest='url_print', action='callback', callback=store_local_curry('canon'))
-    op.add_option('--canonicalize-escape-url', dest='url_print', action='callback', callback=store_local_curry('canon_esc'))
-
-    tunables = [ norm(o.get_opt_string()[2:]) for o in op.option_list if o.callback in (store_abs, 'store_true', None) and o.get_opt_string() not in ('--version', '--help') ]
-    remote_tunables = [ 'listen', 'go_daemon', 'timeout', 'session_owner', 'config_file' ]
-    rq_remote_tunables = { 'listen': True }
-
-    # precedence for sources of values: 1) commandline, 2) cfg file, 3) defaults
-    # -- for this to work out we need to tell apart defaults from explicitly set
-    # options... so churn out the defaults here and call the parser with virgin
-    # values container.
-    defaults = op.get_default_values()
-    opts, args = op.parse_args(values=optparse.Values())
-    confdata = rconf.get('config')
-    if not (len(args) == 2 or \
-            (len(args) == 1 and rconf.get('listen')) or \
-            (len(args) <= 2 and confdata) or \
-            rconf.get('url_print')):
-        sys.stderr.write("error: incorrect number of arguments\n\n")
-        sys.stderr.write(op.get_usage() + "\n")
-        sys.exit(1)
-
-    restricted = os.getenv('_GSYNCD_RESTRICTED_')
-
-    if restricted:
-        allopts = {}
-        allopts.update(opts.__dict__)
-        allopts.update(rconf)
-        bannedtuns = set(allopts.keys()) - set(remote_tunables)
-        if bannedtuns:
-            raise GsyncdError('following tunables cannot be set with restricted SSH invocaton: ' + \
-                              ', '.join(bannedtuns))
-        for k, v in rq_remote_tunables.items():
-            if not k in allopts or allopts[k] != v:
-                raise GsyncdError('tunable %s is not set to value %s required for restricted SSH invocaton' % \
-                                  (k, v))
-
-    confrx = getattr(confdata, 'rx', None)
-    if confrx:
-        # peers are regexen, don't try to parse them
-        if confrx == 'glob':
-            args = [ '\A' + fnmatch.translate(a) for a in args ]
-        canon_peers = args
-        namedict = {}
-    else:
-        rscs = [resource.parse_url(u) for u in args]
-        dc = rconf.get('url_print')
-        if dc:
-            for r in rscs:
-                print(r.get_url(**{'normal': {},
-                                   'canon': {'canonical': True},
-                                   'canon_esc': {'canonical': True, 'escaped': True}}[dc]))
-            return
-        local = remote = None
-        if rscs:
-            local = rscs[0]
-            if len(rscs) > 1:
-                remote = rscs[1]
-            if not local.can_connect_to(remote):
-                raise GsyncdError("%s cannot work with %s" % (local.path, remote and remote.path))
-        pa = ([], [], [])
-        urlprms = ({}, {'canonical': True}, {'canonical': True, 'escaped': True})
-        for x in rscs:
-            for i in range(len(pa)):
-                pa[i].append(x.get_url(**urlprms[i]))
-        peers, canon_peers, canon_esc_peers = pa
-        # creating the namedict, a dict representing various ways of referring to / repreenting
-        # peers to be fillable in config templates
-        mods = (lambda x: x, lambda x: x[0].upper() + x[1:], lambda x: 'e' + x[0].upper() + x[1:])
-        if remote:
-            rmap = { local: ('local', 'master'), remote: ('remote', 'slave') }
-        else:
-            rmap = { local: ('local', 'slave') }
-        namedict = {}
-        for i in range(len(rscs)):
-            x = rscs[i]
-            for name in rmap[x]:
-                for j in range(3):
-                    namedict[mods[j](name)] = pa[j][i]
-                if x.scheme == 'gluster':
-                    namedict[name + 'vol'] = x.volume
-    if not 'config_file' in rconf:
-        rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]), "conf/gsyncd.conf")
-    gcnf = GConffile(rconf['config_file'], canon_peers, defaults.__dict__, opts.__dict__, namedict)
-
-    if confdata:
-        opt_ok = norm(confdata.opt) in tunables + [None]
-        if confdata.op == 'check':
-            if opt_ok:
-                sys.exit(0)
-            else:
-                sys.exit(1)
-        elif not opt_ok:
-            raise GsyncdError("not a valid option: " + confdata.opt)
-        if confdata.op == 'get':
-            gcnf.get(confdata.opt)
-        elif confdata.op == 'set':
-            gcnf.set(confdata.opt, confdata.val, confdata.rx)
-        elif confdata.op == 'del':
-            gcnf.delete(confdata.opt, confdata.rx)
-        return
-
-    gconf.__dict__.update(defaults.__dict__)
-    gcnf.update_to(gconf.__dict__)
-    gconf.__dict__.update(opts.__dict__)
-    gconf.configinterface = gcnf
-
-    if restricted and gconf.allow_network:
-        ssh_conn = os.getenv('SSH_CONNECTION')
-        if not ssh_conn:
-            #legacy env var
-            ssh_conn = os.getenv('SSH_CLIENT')
-        if ssh_conn:
-            allowed_networks = [ IPNetwork(a) for a in gconf.allow_network.split(',') ]
-            client_ip = IPAddress(ssh_conn.split()[0])
-            allowed = False
-            for nw in allowed_networks:
-                if client_ip in nw:
-                    allowed = True
-                    break
-            if not allowed:
-                raise GsyncdError("client IP address is not allowed")
-
-    ffd = rconf.get('feedback_fd')
-    if ffd:
-        fcntl.fcntl(ffd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
-
-    #normalize loglevel
-    lvl0 = gconf.log_level
-    if isinstance(lvl0, str):
-        lvl1 = lvl0.upper()
-        lvl2 = logging.getLevelName(lvl1)
-        # I have _never_ _ever_ seen such an utterly braindead
-        # error condition
-        if lvl2 == "Level " + lvl1:
-            raise GsyncdError('cannot recognize log level "%s"' % lvl0)
-        gconf.log_level = lvl2
-
-    go_daemon = rconf['go_daemon']
-    be_monitor = rconf.get('monitor')
-
-    if not be_monitor and isinstance(remote, resource.SSH) and \
-       go_daemon == 'should':
-        go_daemon = 'postconn'
-        log_file = None
-    else:
-        log_file = gconf.log_file
-    if be_monitor:
-        label = 'monitor'
-    elif remote:
-        #master
-        label = ''
-    else:
-        label = 'slave'
-    startup(go_daemon=go_daemon, log_file=log_file, label=label)
-
-    if be_monitor:
-        return monitor()
-
-    logging.info("syncing: %s" % " -> ".join(peers))
-    resource.Popen.init_errhandler()
-    if remote:
-        go_daemon = remote.connect_remote(go_daemon=go_daemon)
-        if go_daemon:
-            startup(go_daemon=go_daemon, log_file=gconf.log_file)
-            # complete remote connection in child
-            remote.connect_remote(go_daemon='done')
-    local.connect()
-    if ffd:
-        os.close(ffd)
-    local.service_loop(*[r for r in [remote] if r])
-
-
-if __name__ == "__main__":
-    main()
diff --git a/xlators/features/marker/utils/syncdaemon/libcxattr.py b/xlators/features/marker/utils/syncdaemon/libcxattr.py
deleted file mode 100644
index f0a9d2292..000000000
--- a/xlators/features/marker/utils/syncdaemon/libcxattr.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import os
-from ctypes import *
-from ctypes.util import find_library
-
-class Xattr(object):
-    """singleton that wraps the extended attribues system
-       interface for python using ctypes
-
-       Just implement it to the degree we need it, in particular
-       - we need just the l*xattr variants, ie. we never want symlinks to be
-         followed
-       - don't need size discovery for getxattr, as we always know the exact
-         sizes we expect
-    """
-
-    libc = CDLL(find_library("libc"))
-
-    @classmethod
-    def geterrno(cls):
-        return c_int.in_dll(cls.libc, 'errno').value
-
-    @classmethod
-    def raise_oserr(cls):
-        errn = cls.geterrno()
-        raise OSError(errn, os.strerror(errn))
-
-    @classmethod
-    def _query_xattr(cls, path, siz, syscall, *a):
-        if siz:
-            buf = create_string_buffer('\0' * siz)
-        else:
-            buf = None
-        ret = getattr(cls.libc, syscall)(*((path,) + a + (buf, siz)))
-        if ret == -1:
-            cls.raise_oserr()
-        if siz:
-            return buf.raw[:ret]
-        else:
-            return ret
-
-    @classmethod
-    def lgetxattr(cls, path, attr, siz=0):
-        return cls._query_xattr( path, siz, 'lgetxattr', attr)
-
-    @classmethod
-    def llistxattr(cls, path, siz=0):
-        ret = cls._query_xattr(path, siz, 'llistxattr')
-        if isinstance(ret, str):
-            ret = ret.split('\0')
-        return ret
-
-    @classmethod
-    def lsetxattr(cls, path, attr, val):
-        ret = cls.libc.lsetxattr(path, attr, val, len(val), 0)
-        if ret == -1:
-            cls.raise_oserr()
-
-    @classmethod
-    def lremovexattr(cls, path, attr):
-        ret = cls.libc.lremovexattr(path, attr)
-        if ret == -1:
-            cls.raise_oserr()
-
-    @classmethod
-    def llistxattr_buf(cls, path):
-        """listxattr variant with size discovery"""
-        size = cls.llistxattr(path)
-        if size == -1:
-            cls.raise_oserr()
-        if size == 0:
-            return []
-        return cls.llistxattr(path, size)
diff --git a/xlators/features/marker/utils/syncdaemon/master.py b/xlators/features/marker/utils/syncdaemon/master.py
deleted file mode 100644
index cd470f315..000000000
--- a/xlators/features/marker/utils/syncdaemon/master.py
+++ /dev/null
@@ -1,519 +0,0 @@
-import os
-import sys
-import time
-import stat
-import signal
-import logging
-import errno
-from errno import ENOENT, ENODATA
-from threading import currentThread, Condition, Lock
-
-from gconf import gconf
-from syncdutils import FreeObject, Thread, GsyncdError, boolify
-
-URXTIME = (-1, 0)
-
-class GMaster(object):
-    """class impementling master role"""
-
-    KFGN = 0
-    KNAT = 1
-
-    def get_sys_volinfo(self):
-        """query volume marks on fs root
-
-        err out on multiple foreign masters
-        """
-        fgn_vis, nat_vi = self.master.server.foreign_volume_infos(), \
-                          self.master.server.native_volume_info()
-        fgn_vi = None
-        if fgn_vis:
-            if len(fgn_vis) > 1:
-                raise GsyncdError("cannot work with multiple foreign masters")
-            fgn_vi = fgn_vis[0]
-        return fgn_vi, nat_vi
-
-    @property
-    def uuid(self):
-        if self.volinfo:
-            return self.volinfo['uuid']
-
-    @property
-    def volmark(self):
-        if self.volinfo:
-            return self.volinfo['volume_mark']
-
-    @property
-    def inter_master(self):
-        """decide if we are an intermediate master
-        in a cascading setup
-        """
-        return self.volinfo_state[self.KFGN] and True or False
-
-    def xtime(self, path, *a, **opts):
-        """get amended xtime
-
-        as of amending, we can create missing xtime, or
-        determine a valid value if what we get is expired
-        (as of the volume mark expiry); way of amendig
-        depends on @opts and on subject of query (master
-        or slave).
-        """
-        if a:
-            rsc = a[0]
-        else:
-            rsc = self.master
-        if not 'create' in opts:
-            opts['create'] = (rsc == self.master and not self.inter_master)
-        if not 'default_xtime' in opts:
-            if rsc == self.master and self.inter_master:
-                opts['default_xtime'] = ENODATA
-            else:
-                opts['default_xtime'] = URXTIME
-        xt = rsc.server.xtime(path, self.uuid)
-        if isinstance(xt, int) and xt != ENODATA:
-            return xt
-        invalid_xtime = (xt == ENODATA or xt < self.volmark)
-        if invalid_xtime:
-            if opts['create']:
-                t = time.time()
-                sec = int(t)
-                nsec = int((t - sec) * 1000000)
-                xt = (sec, nsec)
-                rsc.server.set_xtime(path, self.uuid, xt)
-            else:
-                xt = opts['default_xtime']
-        return xt
-
-    def __init__(self, master, slave):
-        self.master = master
-        self.slave = slave
-        self.jobtab = {}
-        self.syncer = Syncer(slave)
-        # crawls vs. turns:
-        # - self.crawls is simply the number of crawl() invocations on root
-        # - one turn is a maximal consecutive sequence of crawls so that each
-        #   crawl in it detects a change to be synced
-        # - self.turns is the number of turns since start
-        # - self.total_turns is a limit so that if self.turns reaches it, then
-        #   we exit (for diagnostic purposes)
-        # so, eg., if the master fs changes unceasingly, self.turns will remain 0.
-        self.crawls = 0
-        self.turns = 0
-        self.total_turns = int(gconf.turns)
-        self.lastreport = {'crawls': 0, 'turns': 0}
-        self.start = None
-        self.change_seen = None
-        # the authoritative (foreign, native) volinfo pair
-        # which lets us deduce what to do when we refetch
-        # the volinfos from system
-        uuid_preset = getattr(gconf, 'volume_id', None)
-        self.volinfo_state = (uuid_preset and {'uuid': uuid_preset}, None)
-        # the actual volinfo we make use of
-        self.volinfo = None
-        self.terminate = False
-
-    def crawl_loop(self):
-        """start the keep-alive thread and iterate .crawl"""
-        timo = int(gconf.timeout or 0)
-        if timo > 0:
-            def keep_alive():
-                while True:
-                    gap = timo * 0.5
-                    # first grab a reference as self.volinfo
-                    # can be changed in main thread
-                    vi = self.volinfo
-                    if vi:
-                        # then have a private copy which we can mod
-                        vi = vi.copy()
-                        vi['timeout'] = int(time.time()) + timo
-                    else:
-                        # send keep-alives more frequently to
-                        # avoid a delay in announcing our volume info
-                        # to slave if it becomes established in the
-                        # meantime
-                        gap = min(10, gap)
-                    self.slave.server.keep_alive(vi)
-                    time.sleep(gap)
-            t = Thread(target=keep_alive)
-            t.start()
-        self.lastreport['time'] = time.time()
-        while not self.terminate:
-            self.crawl()
-
-    def add_job(self, path, label, job, *a, **kw):
-        """insert @job function to job table at @path with @label"""
-        if self.jobtab.get(path) == None:
-            self.jobtab[path] = []
-        self.jobtab[path].append((label, a, lambda : job(*a, **kw)))
-
-    def add_failjob(self, path, label):
-        """invoke .add_job with a job that does nothing just fails"""
-        logging.debug('salvaged: ' + label)
-        self.add_job(path, label, lambda: False)
-
-    def wait(self, path, *args):
-        """perform jobs registered for @path
-
-        Reset jobtab entry for @path,
-        determine success as the conjuction of
-        success of all the jobs. In case of
-        success, call .sendmark on @path
-        """
-        jobs = self.jobtab.pop(path, [])
-        succeed = True
-        for j in jobs:
-            ret = j[-1]()
-            if not ret:
-                succeed = False
-        if succeed:
-            self.sendmark(path, *args)
-        return succeed
-
-    def sendmark(self, path, mark, adct=None):
-        """update slave side xtime for @path to master side xtime
-
-        also can send a setattr payload (see Server.setattr).
-        """
-        if adct:
-            self.slave.server.setattr(path, adct)
-        self.slave.server.set_xtime(path, self.uuid, mark)
-
-    @staticmethod
-    def volinfo_state_machine(volinfo_state, volinfo_sys):
-        """compute new volinfo_state from old one and incoming
-           as of current system state, also indicating if there was a
-           change regarding which volume mark is the authoritative one
-
-        @volinfo_state, @volinfo_sys are pairs of volume mark dicts
-        (foreign, native).
-
-        Note this method is marked as static, ie. the computation is
-        pure, without reliance on any excess implicit state. State
-        transitions which are deemed as ambiguous or banned will raise
-        an exception.
-
-        """
-        # store the value below "boxed" to emulate proper closures
-        # (variables of the enclosing scope are available inner functions
-        # provided they are no reassigned; mutation is OK).
-        param = FreeObject(relax_mismatch = False, state_change = None, index=-1)
-        def select_vi(vi0, vi):
-            param.index += 1
-            if vi and (not vi0 or vi0['uuid'] == vi['uuid']):
-                if not vi0 and not param.relax_mismatch:
-                    param.state_change = param.index
-                # valid new value found; for the rest, we are graceful about
-                # uuid mismatch
-                param.relax_mismatch = True
-                return vi
-            if vi0 and vi and vi0['uuid'] != vi['uuid'] and not param.relax_mismatch:
-                # uuid mismatch for master candidate, bail out
-                raise GsyncdError("aborting on uuid change from %s to %s" % \
-                                   (vi0['uuid'], vi['uuid']))
-            # fall back to old
-            return vi0
-        newstate = tuple(select_vi(*vip) for vip in zip(volinfo_state, volinfo_sys))
-        srep = lambda vi: vi and vi['uuid'][0:8]
-        logging.debug('(%s, %s) << (%s, %s) -> (%s, %s)' % \
-                      tuple(srep(vi) for vi in volinfo_state + volinfo_sys + newstate))
-        return newstate, param.state_change
-
-    def crawl(self, path='.', xtl=None):
-        """crawling...
-
-          Standing around
-          All the right people
-          Crawling
-          Tennis on Tuesday
-          The ladder is long
-          It is your nature
-          You've gotta suntan
-          Football on Sunday
-          Society boy
-
-        Recursively walk the master side tree and check if updates are
-        needed due to xtime differences. One invocation of crawl checks
-        children of @path and do a recursive enter only on
-        those directory children where there is an update needed.
-
-        Way of updates depend on file type:
-        - for symlinks, sync them directy and synchronously
-        - for regular children, register jobs for @path (cf. .add_job) to start
-          and wait on their rsync
-        - for directory children, register a job for @path which waits (.wait)
-          on jobs for the given child
-        (other kind of filesystem nodes are not considered)
-
-        Those slave side children which do not exist on master are simply
-        purged (see Server.purge).
-
-        Behavior is fault tolerant, synchronization is adaptive: if some action fails,
-        just go on relentlessly, adding a fail job (see .add_failjob) which will prevent
-        the .sendmark on @path, so when the next crawl will arrive to @path it will not
-        see it as up-to-date and  will try to sync it again. While this semantics can be
-        supported by funky design principles (http://c2.com/cgi/wiki?LazinessImpatienceHubris),
-        the ultimate reason which excludes other possibilities is simply transience: we cannot
-        assert that the file systems (master / slave) underneath do not change and actions
-        taken upon some condition will not lose their context by the time they are performed.
-        """
-        if path == '.':
-            if self.start:
-                self.crawls += 1
-                logging.debug("... crawl #%d done, took %.6f seconds" % \
-                              (self.crawls, time.time() - self.start))
-            time.sleep(1)
-            self.start = time.time()
-            should_display_info = self.start - self.lastreport['time'] >= 60
-            if should_display_info:
-                logging.info("completed %d crawls, %d turns",
-                             self.crawls - self.lastreport['crawls'],
-                             self.turns - self.lastreport['turns'])
-                self.lastreport.update(crawls = self.crawls,
-                                       turns = self.turns,
-                                       time = self.start)
-            volinfo_sys = self.get_sys_volinfo()
-            self.volinfo_state, state_change = self.volinfo_state_machine(self.volinfo_state,
-                                                                          volinfo_sys)
-            if self.inter_master:
-                self.volinfo = volinfo_sys[self.KFGN]
-            else:
-                self.volinfo = volinfo_sys[self.KNAT]
-            if state_change == self.KFGN or (state_change == self.KNAT and not self.inter_master):
-                logging.info('new master is %s', self.uuid)
-                if self.volinfo:
-                    logging.info("%s master with volume id %s ..." % \
-                                 (self.inter_master and "intermediate" or "primary",
-                                  self.uuid))
-            if state_change == self.KFGN:
-               gconf.configinterface.set('volume_id', self.uuid)
-            if self.volinfo:
-                if self.volinfo['retval']:
-                    raise GsyncdError ("master is corrupt")
-            else:
-                if should_display_info or self.crawls == 0:
-                    if self.inter_master:
-                        logging.info("waiting for being synced from %s ..." % \
-                                     self.volinfo_state[self.KFGN]['uuid'])
-                    else:
-                        logging.info("waiting for volume info ...")
-                return
-        logging.debug("entering " + path)
-        if not xtl:
-            xtl = self.xtime(path)
-            if isinstance(xtl, int):
-                self.add_failjob(path, 'no-local-node')
-                return
-        xtr0 = self.xtime(path, self.slave)
-        if isinstance(xtr0, int):
-            if xtr0 != ENOENT:
-                self.slave.server.purge(path)
-            try:
-                self.slave.server.mkdir(path)
-            except OSError:
-                self.add_failjob(path, 'no-remote-node')
-                return
-            xtr = URXTIME
-        else:
-            xtr = xtr0
-            if xtr > xtl:
-                raise GsyncdError("timestamp corruption for " + path)
-            if xtl == xtr:
-                if path == '.' and self.change_seen:
-                    self.turns += 1
-                    self.change_seen = False
-                    if self.total_turns:
-                        logging.info("finished turn #%s/%s" % \
-                                     (self.turns, self.total_turns))
-                        if self.turns == self.total_turns:
-                            logging.info("reached turn limit")
-                            self.terminate = True
-                return
-        if path == '.':
-            self.change_seen = True
-        try:
-            dem = self.master.server.entries(path)
-        except OSError:
-            self.add_failjob(path, 'local-entries-fail')
-            return
-        try:
-            des = self.slave.server.entries(path)
-        except OSError:
-            self.slave.server.purge(path)
-            try:
-                self.slave.server.mkdir(path)
-                des = self.slave.server.entries(path)
-            except OSError:
-                self.add_failjob(path, 'remote-entries-fail')
-                return
-        dd = set(des) - set(dem)
-        if dd and not boolify(gconf.ignore_deletes):
-            self.slave.server.purge(path, dd)
-        chld = []
-        for e in dem:
-            e = os.path.join(path, e)
-            xte = self.xtime(e)
-            if isinstance(xte, int):
-                logging.warn("irregular xtime for %s: %s" % (e, errno.errorcode[xte]))
-            elif xte > xtr:
-                chld.append((e, xte))
-        def indulgently(e, fnc, blame=None):
-            if not blame:
-                blame = path
-            try:
-                return fnc(e)
-            except (IOError, OSError):
-                ex = sys.exc_info()[1]
-                if ex.errno == ENOENT:
-                    logging.warn("salvaged ENOENT for" + e)
-                    self.add_failjob(blame, 'by-indulgently')
-                    return False
-                else:
-                    raise
-        for e, xte in chld:
-            st = indulgently(e, lambda e: os.lstat(e))
-            if st == False:
-                continue
-            mo = st.st_mode
-            adct = {'own': (st.st_uid, st.st_gid)}
-            if stat.S_ISLNK(mo):
-                if indulgently(e, lambda e: self.slave.server.symlink(os.readlink(e), e)) == False:
-                    continue
-                self.sendmark(e, xte, adct)
-            elif stat.S_ISREG(mo):
-                logging.debug("syncing %s ..." % e)
-                pb = self.syncer.add(e)
-                def regjob(e, xte, pb):
-                    if pb.wait():
-                        logging.debug("synced " + e)
-                        self.sendmark(e, xte)
-                        return True
-                    else:
-                        logging.error("failed to sync " + e)
-                self.add_job(path, 'reg', regjob, e, xte, pb)
-            elif stat.S_ISDIR(mo):
-                adct['mode'] = mo
-                if indulgently(e, lambda e: (self.add_job(path, 'cwait', self.wait, e, xte, adct),
-                                             self.crawl(e, xte),
-                                             True)[-1], blame=e) == False:
-                    continue
-            else:
-                # ignore fifos, sockets and special files
-                pass
-        if path == '.':
-            self.wait(path, xtl)
-
-class BoxClosedErr(Exception):
-    pass
-
-class PostBox(list):
-    """synchronized collection for storing things thought of as "requests" """
-
-    def __init__(self, *a):
-        list.__init__(self, *a)
-        # too bad Python stdlib does not have read/write locks...
-        # it would suffivce to grab the lock in .append as reader, in .close as writer
-        self.lever = Condition()
-        self.open = True
-        self.done = False
-
-    def wait(self):
-        """wait on requests to be processed"""
-        self.lever.acquire()
-        if not self.done:
-            self.lever.wait()
-        self.lever.release()
-        return self.result
-
-    def wakeup(self, data):
-        """wake up requestors with the result"""
-        self.result = data
-        self.lever.acquire()
-        self.done = True
-        self.lever.notifyAll()
-        self.lever.release()
-
-    def append(self, e):
-        """post a request"""
-        self.lever.acquire()
-        if not self.open:
-            raise BoxClosedErr
-        list.append(self, e)
-        self.lever.release()
-
-    def close(self):
-        """prohibit the posting of further requests"""
-        self.lever.acquire()
-        self.open = False
-        self.lever.release()
-
-class Syncer(object):
-    """a staged queue to relay rsync requests to rsync workers
-
-    By "staged queue" its meant that when a consumer comes to the
-    queue, it takes _all_ entries, leaving the queue empty.
-    (I don't know if there is an official term for this pattern.)
-
-    The queue uses a PostBox to accumulate incoming items.
-    When a consumer (rsync worker) comes, a new PostBox is
-    set up and the old one is passed on to the consumer.
-
-    Instead of the simplistic scheme of having one big lock
-    which synchronizes both the addition of new items and
-    PostBox exchanges, use a separate lock to arbitrate consumers,
-    and rely on PostBox's synchronization mechanisms take
-    care about additions.
-
-    There is a corner case racy situation, producers vs. consumers,
-    which is not handled by this scheme: namely, when the PostBox
-    exchange occurs in between being passed to the producer for posting
-    and the post placement. But that's what Postbox.close is for:
-    such a posting will find the PostBox closed, in which case
-    the producer can re-try posting against the actual PostBox of
-    the queue.
-
-    To aid accumlation of items in the PostBoxen before grabbed
-    by an rsync worker, the worker goes to sleep a bit after
-    each completed syncjob.
-    """
-
-    def __init__(self, slave):
-        """spawn worker threads"""
-        self.slave = slave
-        self.lock = Lock()
-        self.pb = PostBox()
-        for i in range(int(gconf.sync_jobs)):
-            t = Thread(target=self.syncjob)
-            t.start()
-
-    def syncjob(self):
-        """the life of a worker"""
-        while True:
-            pb = None
-            while True:
-                self.lock.acquire()
-                if self.pb:
-                    pb, self.pb = self.pb, PostBox()
-                self.lock.release()
-                if pb:
-                    break
-                time.sleep(0.5)
-            pb.close()
-            po = self.slave.rsync(pb)
-            if po.returncode == 0:
-                ret = True
-            elif po.returncode in (23, 24):
-                # partial transfer (cf. rsync(1)), that's normal
-                ret = False
-            else:
-                po.errfail()
-            pb.wakeup(ret)
-
-    def add(self, e):
-        while True:
-            pb = self.pb
-            try:
-                pb.append(e)
-                return pb
-            except BoxClosedErr:
-                pass
diff --git a/xlators/features/marker/utils/syncdaemon/monitor.py b/xlators/features/marker/utils/syncdaemon/monitor.py
deleted file mode 100644
index 981643ec0..000000000
--- a/xlators/features/marker/utils/syncdaemon/monitor.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import os
-import sys
-import time
-import signal
-import logging
-from gconf import gconf
-from syncdutils import update_file, select, waitpid
-
-class Monitor(object):
-    """class which spawns and manages gsyncd workers"""
-
-    def __init__(self):
-        self.state = None
-
-    def set_state(self, state):
-        """set the state that can be used by external agents
-           like glusterd for status reporting"""
-        if state == self.state:
-            return
-        self.state = state
-        logging.info('new state: %s' % state)
-        if getattr(gconf, 'state_file', None):
-            update_file(gconf.state_file, lambda f: f.write(state + '\n'))
-
-    def monitor(self):
-        """the monitor loop
-
-        Basic logic is a blantantly simple blunt heuristics:
-        if spawned client survives 60 secs, it's considered OK.
-        This servers us pretty well as it's not vulneralbe to
-        any kind of irregular behavior of the child...
-
-        ... well, except for one: if children is hung up on
-        waiting for some event, it can survive aeons, still
-        will be defunct. So we tweak the above logic to
-        expect the worker to send us a signal within 60 secs
-        (in the form of closing its end of a pipe). The worker
-        does this when it's done with the setup stage
-        ready to enter the service loop (note it's the setup
-        stage which is vulnerable to hangs -- the full
-        blown worker blows up on EPIPE if the net goes down,
-        due to the keep-alive thread)
-        """
-        def sigcont_handler(*a):
-            """
-            Re-init logging and send group kill signal
-            """
-            md = gconf.log_metadata
-            logging.shutdown()
-            lcls = logging.getLoggerClass()
-            lcls.setup(label=md.get('saved_label'), **md)
-            pid = os.getpid()
-            os.kill(-pid, signal.SIGUSR1)
-        signal.signal(signal.SIGUSR1, lambda *a: ())
-        signal.signal(signal.SIGCONT, sigcont_handler)
-
-        argv = sys.argv[:]
-        for o in ('-N', '--no-daemon', '--monitor'):
-            while o in argv:
-                argv.remove(o)
-        argv.extend(('-N', '-p', ''))
-        argv.insert(0, os.path.basename(sys.executable))
-
-        self.set_state('starting...')
-        ret = 0
-        def nwait(p, o=0):
-            p2, r = waitpid(p, o)
-            if not p2:
-                return
-            return r
-        def exit_signalled(s):
-            """ child teminated due to receipt of SIGUSR1 """
-            return (os.WIFSIGNALED(s) and (os.WTERMSIG(s) == signal.SIGUSR1))
-        def exit_status(s):
-            if os.WIFEXITED(s):
-                return os.WEXITSTATUS(s)
-            return 1
-        conn_timeout = int(gconf.connection_timeout)
-        while ret in (0, 1):
-            logging.info('-' * conn_timeout)
-            logging.info('starting gsyncd worker')
-            pr, pw = os.pipe()
-            cpid = os.fork()
-            if cpid == 0:
-                os.close(pr)
-                os.execv(sys.executable, argv + ['--feedback-fd', str(pw)])
-            os.close(pw)
-            t0 = time.time()
-            so = select((pr,), (), (), conn_timeout)[0]
-            os.close(pr)
-            if so:
-                ret = nwait(cpid, os.WNOHANG)
-                if ret != None:
-                    logging.debug("worker died before establishing connection")
-                else:
-                    logging.debug("worker seems to be connected (?? racy check)")
-                    while time.time() < t0 + conn_timeout:
-                        ret = nwait(cpid, os.WNOHANG)
-                        if ret != None:
-                            logging.debug("worker died in startup phase")
-                            break
-                        time.sleep(1)
-            else:
-                logging.debug("worker not confirmed in %d sec, aborting it" % \
-                              conn_timeout)
-                os.kill(cpid, signal.SIGKILL)
-                ret = nwait(cpid)
-            if ret == None:
-                self.set_state('OK')
-                ret = nwait(cpid)
-            if exit_signalled(ret):
-                ret = 0
-            else:
-                ret = exit_status(ret)
-                if ret in (0,1):
-                    self.set_state('faulty')
-            time.sleep(10)
-        self.set_state('inconsistent')
-        return ret
-
-def monitor():
-    """oh yeah, actually Monitor is used as singleton, too"""
-    return Monitor().monitor()
diff --git a/xlators/features/marker/utils/syncdaemon/repce.py b/xlators/features/marker/utils/syncdaemon/repce.py
deleted file mode 100644
index 755fb61df..000000000
--- a/xlators/features/marker/utils/syncdaemon/repce.py
+++ /dev/null
@@ -1,225 +0,0 @@
-import os
-import sys
-import time
-import logging
-from threading import Condition
-try:
-    import thread
-except ImportError:
-    # py 3
-    import _thread as thread
-try:
-    from Queue import Queue
-except ImportError:
-    # py 3
-    from queue import Queue
-try:
-    import cPickle as pickle
-except ImportError:
-    # py 3
-    import pickle
-
-from syncdutils import Thread, select
-
-pickle_proto = -1
-repce_version = 1.0
-
-def ioparse(i, o):
-    if isinstance(i, int):
-        i = os.fdopen(i)
-    # rely on duck typing for recognizing
-    # streams as that works uniformly
-    # in py2 and py3
-    if hasattr(o, 'fileno'):
-        o = o.fileno()
-    return (i, o)
-
-def send(out, *args):
-    """pickle args and write out wholly in one syscall
-
-    ie. not use the ability of pickle to dump directly to
-    a stream, as that would potentially mess up messages
-    by interleaving them
-    """
-    os.write(out, pickle.dumps(args, pickle_proto))
-
-def recv(inf):
-    """load an object from input stream"""
-    return pickle.load(inf)
-
-
-class RepceServer(object):
-    """RePCe is Hungarian for canola, http://hu.wikipedia.org/wiki/Repce
-
-    ... also our homebrewed RPC backend where the transport layer is
-    reduced to a pair of filehandles.
-
-    This is the server component.
-    """
-
-    def __init__(self, obj, i, o, wnum=6):
-        """register a backend object .obj to which incoming messages
-           are dispatched, also incoming/outcoming streams
-        """
-        self.obj = obj
-        self.inf, self.out = ioparse(i, o)
-        self.wnum = wnum
-        self.q = Queue()
-
-    def service_loop(self):
-        """fire up worker threads, get messages and dispatch among them"""
-        for i in range(self.wnum):
-            t = Thread(target=self.worker)
-            t.start()
-        try:
-            while True:
-                self.q.put(recv(self.inf))
-        except EOFError:
-            logging.info("terminating on reaching EOF.")
-
-    def worker(self):
-        """life of a worker
-
-        Get message, extract its id, method name and arguments
-        (kwargs not supported), call method on .obj.
-        Send back message id + return value.
-        If method call throws an exception, rescue it, and send
-        back the exception as result (with flag marking it as
-        exception).
-        """
-        while True:
-            in_data = self.q.get(True)
-            rid = in_data[0]
-            rmeth = in_data[1]
-            exc = False
-            if rmeth == '__repce_version__':
-                res = repce_version
-            else:
-              try:
-                  res = getattr(self.obj, rmeth)(*in_data[2:])
-              except:
-                  res = sys.exc_info()[1]
-                  exc = True
-                  logging.exception("call failed: ")
-            send(self.out, rid, exc, res)
-
-
-class RepceJob(object):
-    """class representing message status we can use
-    for waiting on reply"""
-
-    def __init__(self, cbk):
-        """
-        - .rid: (process-wise) unique id
-        - .cbk: what we do upon receiving reply
-        """
-        self.rid = (os.getpid(), thread.get_ident(), time.time())
-        self.cbk = cbk
-        self.lever = Condition()
-        self.done = False
-
-    def __repr__(self):
-        return ':'.join([str(x) for x in self.rid])
-
-    def wait(self):
-        self.lever.acquire()
-        if not self.done:
-            self.lever.wait()
-        self.lever.release()
-        return self.result
-
-    def wakeup(self, data):
-        self.result = data
-        self.lever.acquire()
-        self.done = True
-        self.lever.notify()
-        self.lever.release()
-
-
-class RepceClient(object):
-    """RePCe is Hungarian for canola, http://hu.wikipedia.org/wiki/Repce
-
-    ... also our homebrewed RPC backend where the transport layer is
-    reduced to a pair of filehandles.
-
-    This is the client component.
-    """
-
-    def __init__(self, i, o):
-        self.inf, self.out = ioparse(i, o)
-        self.jtab = {}
-        t = Thread(target = self.listen)
-        t.start()
-
-    def listen(self):
-        while True:
-            select((self.inf,), (), ())
-            rid, exc, res = recv(self.inf)
-            rjob = self.jtab.pop(rid)
-            if rjob.cbk:
-                rjob.cbk(rjob, [exc, res])
-
-    def push(self, meth, *args, **kw):
-        """wrap arguments in a RepceJob, send them to server
-           and return the RepceJob
-
-           @cbk to pass on RepceJob can be given as kwarg.
-        """
-        cbk = kw.get('cbk')
-        if not cbk:
-            def cbk(rj, res):
-                if res[0]:
-                    raise res[1]
-        rjob = RepceJob(cbk)
-        self.jtab[rjob.rid] = rjob
-        logging.debug("call %s %s%s ..." % (repr(rjob), meth, repr(args)))
-        send(self.out, rjob.rid, meth, *args)
-        return rjob
-
-    def __call__(self, meth, *args):
-        """RePCe client is callabe, calling it implements a synchronous remote call
-
-        We do a .push with a cbk which does a wakeup upon receiving anwser, then wait
-        on the RepceJob.
-        """
-        rjob = self.push(meth, *args, **{'cbk': lambda rj, res: rj.wakeup(res)})
-        exc, res = rjob.wait()
-        if exc:
-            logging.error('call %s (%s) failed on peer with %s' % (repr(rjob), meth, str(type(res).__name__)))
-            raise res
-        logging.debug("call %s %s -> %s" % (repr(rjob), meth, repr(res)))
-        return res
-
-    class mprx(object):
-        """method proxy, standard trick to implement rubyesque method_missing
-           in Python
-
-        A class is a closure factory, you know what I mean, or go read some SICP.
-        """
-        def __init__(self, ins, meth):
-            self.ins = ins
-            self.meth = meth
-
-        def __call__(self, *a):
-            return self.ins(self.meth, *a)
-
-    def __getattr__(self, meth):
-        """this implements transparent method dispatch to remote object,
-           so that you don't need to call the RepceClient instance like
-
-             rclient('how_old_are_you_if_born_in', 1979)
-
-           but you can make it into an ordinary method call like
-
-             rclient.how_old_are_you_if_born_in(1979)
-        """
-        return self.mprx(self, meth)
-
-    def __version__(self):
-        """used in handshake to verify compatibility"""
-        d = {'proto': self('__repce_version__')}
-        try:
-            d['object'] = self('version')
-        except AttributeError:
-            pass
-        return d
diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/xlators/features/marker/utils/syncdaemon/resource.py
deleted file mode 100644
index 7fcc53964..000000000
--- a/xlators/features/marker/utils/syncdaemon/resource.py
+++ /dev/null
@@ -1,835 +0,0 @@
-import re
-import os
-import sys
-import stat
-import time
-import errno
-import struct
-import socket
-import logging
-import tempfile
-import threading
-import subprocess
-from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR
-
-from gconf import gconf
-import repce
-from repce import RepceServer, RepceClient
-from master import GMaster
-import syncdutils
-from syncdutils import GsyncdError, select, privileged
-
-UrlRX  = re.compile('\A(\w+)://([^ *?[]*)\Z')
-HostRX = re.compile('[a-z\d](?:[a-z\d.-]*[a-z\d])?', re.I)
-UserRX = re.compile("[\w!\#$%&'*+-\/=?^_`{|}~]+")
-
-def sup(x, *a, **kw):
-    """a rubyesque "super" for python ;)
-
-    invoke caller method in parent class with given args.
-    """
-    return getattr(super(type(x), x), sys._getframe(1).f_code.co_name)(*a, **kw)
-
-def desugar(ustr):
-    """transform sugared url strings to standard <scheme>://<urlbody> form
-
-    parsing logic enforces the constraint that sugared forms should contatin
-    a ':' or a '/', which ensures that sugared urls do not conflict with
-    gluster volume names.
-    """
-    m = re.match('([^:]*):(.*)', ustr)
-    if m:
-        if not m.groups()[0]:
-            return "gluster://localhost" + ustr
-        elif '@' in m.groups()[0] or re.search('[:/]', m.groups()[1]):
-            return "ssh://" + ustr
-        else:
-            return "gluster://" + ustr
-    else:
-        if ustr[0] != '/':
-            raise GsyncdError("cannot resolve sugared url '%s'" % ustr)
-        ap = os.path.normpath(ustr)
-        if ap.startswith('//'):
-            ap = ap[1:]
-        return "file://" + ap
-
-def gethostbyname(hnam):
-    """gethostbyname wrapper"""
-    try:
-        return socket.gethostbyname(hnam)
-    except socket.gaierror:
-        ex = sys.exc_info()[1]
-        raise GsyncdError("failed to resolve %s: %s" % \
-                          (hnam, ex.strerror))
-
-def parse_url(ustr):
-    """instantiate an url object by scheme-to-class dispatch
-
-    The url classes taken into consideration are the ones in
-    this module whose names are full-caps.
-    """
-    m = UrlRX.match(ustr)
-    if not m:
-        ustr = desugar(ustr)
-    m = UrlRX.match(ustr)
-    if not m:
-        raise GsyncdError("malformed url")
-    sch, path = m.groups()
-    this = sys.modules[__name__]
-    if not hasattr(this, sch.upper()):
-        raise GsyncdError("unknown url scheme " + sch)
-    return getattr(this, sch.upper())(path)
-
-
-class _MetaXattr(object):
-    """singleton class, a lazy wrapper around the
-    libcxattr module
-
-    libcxattr (a heavy import due to ctypes) is
-    loaded only when when the single
-    instance is tried to be used.
-
-    This reduces runtime for those invocations
-    which do not need filesystem manipulation
-    (eg. for config, url parsing)
-    """
-
-    def __getattr__(self, meth):
-        from libcxattr import Xattr as LXattr
-        xmeth = [ m for m in dir(LXattr) if m[0] != '_' ]
-        if not meth in xmeth:
-            return
-        for m in xmeth:
-            setattr(self, m, getattr(LXattr, m))
-        return getattr(self, meth)
-
-Xattr = _MetaXattr()
-
-
-class Popen(subprocess.Popen):
-    """customized subclass of subprocess.Popen with a ring
-    buffer for children error ouput"""
-
-    @classmethod
-    def init_errhandler(cls):
-        """start the thread which handles children's error output"""
-        cls.errstore = {}
-        def tailer():
-            while True:
-                for po in select([po.stderr for po in cls.errstore], [], []):
-                    po.lock.acquire()
-                    try:
-                        la = cls.errstore.get(po)
-                        if la == None:
-                            continue
-                        l = os.read(po.stderr.fileno(), 1024)
-                        tots = len(l)
-                        for lx in la:
-                            tots += len(lx)
-                        while tots > 1<<20 and la:
-                            tots -= len(la.pop(0))
-                    finally:
-                        po.lock.release()
-        t = syncdutils.Thread(target = tailer)
-        t.start()
-        cls.errhandler = t
-
-    def __init__(self, args, *a, **kw):
-        """customizations for subprocess.Popen instantiation
-
-        - 'close_fds' is taken to be the default
-        - if child's stderr is chosen to be managed,
-          register it with the error handler thread
-        """
-        self.args = args
-        if 'close_fds' not in kw:
-            kw['close_fds'] = True
-        self.lock = threading.Lock()
-        try:
-            sup(self, args, *a, **kw)
-        except:
-            ex = sys.exc_info()[1]
-            if not isinstance(ex, OSError):
-                raise
-            raise GsyncdError("""execution of "%s" failed with %s (%s)""" % \
-                              (args[0], errno.errorcode[ex.errno], os.strerror(ex.errno)))
-        if kw['stderr'] == subprocess.PIPE:
-            assert(getattr(self, 'errhandler', None))
-            self.errstore[self] = []
-
-    def errfail(self):
-        """fail nicely if child did not terminate with success"""
-        filling = None
-        if self.elines:
-            filling = ", saying:"
-        logging.error("""command "%s" returned with %d%s""" % \
-                      (" ".join(self.args), self.returncode, filling))
-        for l in self.elines:
-            for ll in l.rstrip().split("\n"):
-                logging.error(self.args[0] + "> " + ll.rstrip())
-        syncdutils.finalize(exval = 1)
-
-    def terminate_geterr(self, fail_on_err = True):
-        """kill child, finalize stderr harvesting (unregister
-        from errhandler, set up .elines), fail on error if
-        asked for
-        """
-        self.lock.acquire()
-        try:
-            elines = self.errstore.pop(self)
-        finally:
-            self.lock.release()
-        if self.poll() == None:
-            self.terminate()
-            if sp.poll() == None:
-                time.sleep(0.1)
-            sp.kill()
-        while True:
-            b = os.read(self.stderr.fileno(), 1024)
-            if b:
-                elines.append(b)
-            else:
-                break
-        self.stderr.close()
-        self.elines = elines
-        if fail_on_err and self.returncode != 0:
-            self.errfail()
-
-
-class Server(object):
-    """singleton implemening those filesystem access primitives
-       which are needed for geo-replication functionality
-
-    (Singleton in the sense it's a class which has only static
-    and classmethods and is used directly, without instantiation.)
-    """
-
-    GX_NSPACE = (privileged() and "trusted" or "system") + ".glusterfs"
-    NTV_FMTSTR = "!" + "B"*19 + "II"
-    FRGN_XTRA_FMT = "I"
-    FRGN_FMTSTR = NTV_FMTSTR + FRGN_XTRA_FMT
-
-    def _pathguard(f):
-        """decorator method that checks
-        the path argument of the decorated
-        functions to make sure it does not
-        point out of the managed tree
-        """
-
-        fc = getattr(f, 'func_code', None)
-        if not fc:
-            # python 3
-            fc = f.__code__
-        pi = list(fc.co_varnames).index('path')
-        def ff(*a):
-            path = a[pi]
-            ps = path.split('/')
-            if path[0] == '/' or '..' in ps:
-                raise ValueError('unsafe path')
-            return f(*a)
-        return ff
-
-    @staticmethod
-    @_pathguard
-    def entries(path):
-        """directory entries in an array"""
-        # prevent symlinks being followed
-        if not stat.S_ISDIR(os.lstat(path).st_mode):
-            raise OSError(ENOTDIR, os.strerror(ENOTDIR))
-        return os.listdir(path)
-
-    @classmethod
-    @_pathguard
-    def purge(cls, path, entries=None):
-        """force-delete subtrees
-
-        If @entries is not specified, delete
-        the whole subtree under @path (including
-        @path).
-
-        Otherwise, @entries should be a
-        a sequence of children of @path, and
-        the effect is identical with a joint
-        @entries-less purge on them, ie.
-
-        for e in entries:
-            cls.purge(os.path.join(path, e))
-        """
-        me_also = entries == None
-        if not entries:
-            try:
-                # if it's a symlink, prevent
-                # following it
-                try:
-                    os.unlink(path)
-                    return
-                except OSError:
-                    ex = sys.exc_info()[1]
-                    if ex.errno == EISDIR:
-                        entries = os.listdir(path)
-                    else:
-                        raise
-            except OSError:
-                ex = sys.exc_info()[1]
-                if ex.errno in (ENOTDIR, ENOENT, ELOOP):
-                    try:
-                        os.unlink(path)
-                        return
-                    except OSError:
-                        ex = sys.exc_info()[1]
-                        if ex.errno == ENOENT:
-                            return
-                        raise
-                else:
-                    raise
-        for e in entries:
-            cls.purge(os.path.join(path, e))
-        if me_also:
-            os.rmdir(path)
-
-    @classmethod
-    @_pathguard
-    def _create(cls, path, ctor):
-        """path creation backend routine"""
-        try:
-            ctor(path)
-        except OSError:
-            ex = sys.exc_info()[1]
-            if ex.errno == EEXIST:
-                cls.purge(path)
-                return ctor(path)
-            raise
-
-    @classmethod
-    @_pathguard
-    def mkdir(cls, path):
-        cls._create(path, os.mkdir)
-
-    @classmethod
-    @_pathguard
-    def symlink(cls, lnk, path):
-        cls._create(path, lambda p: os.symlink(lnk, p))
-
-    @classmethod
-    @_pathguard
-    def xtime(cls, path, uuid):
-        """query xtime extended attribute
-
-        Return xtime of @path for @uuid as a pair of integers.
-        "Normal" errors due to non-existent @path or extended attribute
-        are tolerated and errno is returned in such a case.
-        """
-
-        try:
-            return struct.unpack('!II', Xattr.lgetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), 8))
-        except OSError:
-            ex = sys.exc_info()[1]
-            if ex.errno in (ENOENT, ENODATA, ENOTDIR):
-                return ex.errno
-            else:
-                raise
-
-    @classmethod
-    @_pathguard
-    def set_xtime(cls, path, uuid, mark):
-        """set @mark as xtime for @uuid on @path"""
-        Xattr.lsetxattr(path, '.'.join([cls.GX_NSPACE, uuid, 'xtime']), struct.pack('!II', *mark))
-
-    @staticmethod
-    @_pathguard
-    def setattr(path, adct):
-        """set file attributes
-
-        @adct is a dict, where 'own', 'mode' and 'times'
-        keys are looked for and values used to perform
-        chown, chmod or utimes on @path.
-        """
-        own = adct.get('own')
-        if own:
-            os.lchown(path, *own)
-        mode = adct.get('mode')
-        if mode:
-            os.chmod(path, stat.S_IMODE(mode))
-        times = adct.get('times')
-        if times:
-            os.utime(path, times)
-
-    @staticmethod
-    def pid():
-        return os.getpid()
-
-    last_keep_alive = 0
-    @classmethod
-    def keep_alive(cls, dct):
-        """process keepalive messages.
-
-        Return keep-alive counter (number of received keep-alive
-        messages).
-
-        Now the "keep-alive" message can also have a payload which is
-        used to set a foreign volume-mark on the underlying file system.
-        """
-        if dct:
-            key = '.'.join([cls.GX_NSPACE, 'volume-mark', dct['uuid']])
-            val = struct.pack(cls.FRGN_FMTSTR,
-                              *(dct['version']  +
-                                tuple(int(x,16) for x in re.findall('(?:[\da-f]){2}', dct['uuid'])) +
-                                (dct['retval'],) + dct['volume_mark'][0:2] + (dct['timeout'],)))
-            Xattr.lsetxattr('.', key, val)
-        cls.last_keep_alive += 1
-        return cls.last_keep_alive
-
-    @staticmethod
-    def version():
-        """version used in handshake"""
-        return 1.0
-
-
-class SlaveLocal(object):
-    """mix-in class to implement some factes of a slave server
-
-    ("mix-in" is sort of like "abstract class", ie. it's not
-    instantiated just included in the ancesty DAG. I use "mix-in"
-    to indicate that it's not used as an abstract base class,
-    rather just taken in to implement additional functionality
-    on the basis of the assumed availability of certain interfaces.)
-    """
-
-    def can_connect_to(self, remote):
-        """determine our position in the connectibility matrix"""
-        return not remote
-
-    def service_loop(self):
-        """start a RePCe server serving self's server
-
-        stop servicing if a timeout is configured and got no
-        keep-alime in that inteval
-        """
-        repce = RepceServer(self.server, sys.stdin, sys.stdout, int(gconf.sync_jobs))
-        t = syncdutils.Thread(target=lambda: (repce.service_loop(),
-                                              syncdutils.finalize()))
-        t.start()
-        logging.info("slave listening")
-        if gconf.timeout and int(gconf.timeout) > 0:
-            while True:
-                lp = self.server.last_keep_alive
-                time.sleep(int(gconf.timeout))
-                if lp == self.server.last_keep_alive:
-                    logging.info("connection inactive for %d seconds, stopping" % int(gconf.timeout))
-                    break
-        else:
-            select((), (), ())
-
-class SlaveRemote(object):
-    """mix-in class to implement an interface to a remote slave"""
-
-    def connect_remote(self, rargs=[], **opts):
-        """connects to a remote slave
-
-        Invoke an auxiliary utility (slave gsyncd, possibly wrapped)
-        which sets up the connection and set up a RePCe client to
-        communicate throuh its stdio.
-        """
-        slave = opts.get('slave', self.url)
-        so = getattr(gconf, 'session_owner', None)
-        if so:
-            so_args = ['--session-owner', so]
-        else:
-            so_args = []
-        po = Popen(rargs + gconf.remote_gsyncd.split() + so_args + \
-                   ['-N', '--listen', '--timeout', str(gconf.timeout), slave],
-                   stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        gconf.transport = po
-        return self.start_fd_client(po.stdout, po.stdin, **opts)
-
-    def start_fd_client(self, i, o, **opts):
-        """set up RePCe client, handshake with server
-
-        It's cut out as a separate method to let
-        subclasses hook into client startup
-        """
-        self.server = RepceClient(i, o)
-        rv = self.server.__version__()
-        exrv = {'proto': repce.repce_version, 'object': Server.version()}
-        da0 = (rv, exrv)
-        da1 = ({}, {})
-        for i in range(2):
-            for k, v in da0[i].iteritems():
-                da1[i][k] = int(v)
-        if da1[0] != da1[1]:
-            raise GsyncdError("RePCe major version mismatch: local %s, remote %s" % (exrv, rv))
-
-    def rsync(self, files, *args):
-        """invoke rsync"""
-        if not files:
-            raise GsyncdError("no files to sync")
-        logging.debug("files: " + ", ".join(files))
-        argv = gconf.rsync_command.split() + gconf.rsync_extra.split() + ['-aR'] + files + list(args)
-        po = Popen(argv, stderr=subprocess.PIPE)
-        po.wait()
-        po.terminate_geterr(fail_on_err = False)
-        return po
-
-
-class AbstractUrl(object):
-    """abstract base class for url scheme classes"""
-
-    def __init__(self, path, pattern):
-        m = re.search(pattern, path)
-        if not m:
-            raise GsyncdError("malformed path")
-        self.path = path
-        return m.groups()
-
-    @property
-    def scheme(self):
-        return type(self).__name__.lower()
-
-    def canonical_path(self):
-        return self.path
-
-    def get_url(self, canonical=False, escaped=False):
-        """format self's url in various styles"""
-        if canonical:
-            pa = self.canonical_path()
-        else:
-            pa = self.path
-        u = "://".join((self.scheme, pa))
-        if escaped:
-            u = syncdutils.escape(u)
-        return u
-
-    @property
-    def url(self):
-        return self.get_url()
-
-
-  ### Concrete resource classes ###
-
-
-class FILE(AbstractUrl, SlaveLocal, SlaveRemote):
-    """scheme class for file:// urls
-
-    can be used to represent a file slave server
-    on slave side, or interface to a remote file
-    file server on master side
-    """
-
-    class FILEServer(Server):
-        """included server flavor"""
-        pass
-
-    server = FILEServer
-
-    def __init__(self, path):
-        sup(self, path, '^/')
-
-    def connect(self):
-        """inhibit the resource beyond"""
-        os.chdir(self.path)
-
-    def rsync(self, files):
-        return sup(self, files, self.path)
-
-
-class GLUSTER(AbstractUrl, SlaveLocal, SlaveRemote):
-    """scheme class for gluster:// urls
-
-    can be used to represent a gluster slave server
-    on slave side, or interface to a remote gluster
-    slave on master side, or to represent master
-    (slave-ish features come from the mixins, master
-    functionality is outsourced to GMaster from master)
-    """
-
-    class GLUSTERServer(Server):
-        "server enhancements for a glusterfs backend"""
-
-        @classmethod
-        def _attr_unpack_dict(cls, xattr, extra_fields = ''):
-            """generic volume mark fetching/parsing backed"""
-            fmt_string = cls.NTV_FMTSTR + extra_fields
-            buf = Xattr.lgetxattr('.', xattr, struct.calcsize(fmt_string))
-            vm = struct.unpack(fmt_string, buf)
-            m = re.match('(.{8})(.{4})(.{4})(.{4})(.{12})', "".join(['%02x' % x for x in vm[2:18]]))
-            uuid = '-'.join(m.groups())
-            volinfo = {  'version': vm[0:2],
-                         'uuid'   : uuid,
-                         'retval' : vm[18],
-                         'volume_mark': vm[19:21],
-                      }
-            if extra_fields:
-                return volinfo, vm[-len(extra_fields):]
-            else:
-                return volinfo
-
-        @classmethod
-        def foreign_volume_infos(cls):
-            """return list of valid (not expired) foreign volume marks"""
-            dict_list = []
-            xattr_list = Xattr.llistxattr_buf('.')
-            for ele in xattr_list:
-                if ele.find('.'.join([cls.GX_NSPACE, 'volume-mark', ''])) == 0:
-                    d, x = cls._attr_unpack_dict(ele, cls.FRGN_XTRA_FMT)
-                    now = int(time.time())
-                    if x[0] > now:
-                        logging.debug("volinfo[%s] expires: %d (%d sec later)" % \
-                                      (d['uuid'], x[0], x[0] - now))
-                        dict_list.append(d)
-                    else:
-                        try:
-                            Xattr.lremovexattr('.', ele)
-                        except OSError:
-                            pass
-            return dict_list
-
-        @classmethod
-        def native_volume_info(cls):
-            """get the native volume mark of the underlying gluster volume"""
-            try:
-                return cls._attr_unpack_dict('.'.join([cls.GX_NSPACE, 'volume-mark']))
-            except OSError:
-                ex = sys.exc_info()[1]
-                if ex.errno != ENODATA:
-                    raise
-
-    server = GLUSTERServer
-
-    def __init__(self, path):
-        self.host, self.volume = sup(self, path, '^(%s):(.+)' % HostRX.pattern)
-
-    def canonical_path(self):
-        return ':'.join([gethostbyname(self.host), self.volume])
-
-    def can_connect_to(self, remote):
-        """determine our position in the connectibility matrix"""
-        return True
-
-    class Mounter(object):
-        """Abstract base class for mounter backends"""
-
-        def __init__(self, params):
-            self.params = params
-
-        @classmethod
-        def get_glusterprog(cls):
-            return os.path.join(gconf.gluster_command_dir, cls.glusterprog)
-
-        def umount_l(self, d):
-            """perform lazy umount"""
-            po = Popen(self.make_umount_argv(d), stderr=subprocess.PIPE)
-            po.wait()
-            return po
-
-        @classmethod
-        def make_umount_argv(cls, d):
-            raise NotImplementedError
-
-        def make_mount_argv(self, *a):
-            raise NotImplementedError
-
-        def cleanup_mntpt(self):
-            pass
-
-        def handle_mounter(self, po):
-            po.wait()
-
-        def inhibit(self, *a):
-            """inhibit a gluster filesystem
-
-            Mount glusterfs over a temporary mountpoint,
-            change into the mount, and lazy unmount the
-            filesystem.
-            """
-            mounted = False
-            try:
-                po = Popen(self.make_mount_argv(*a), **self.mountkw)
-                self.handle_mounter(po)
-                po.terminate_geterr()
-                d = self.mntpt
-                mounted = True
-                logging.debug('auxiliary glusterfs mount in place')
-                os.chdir(d)
-                self.umount_l(d).terminate_geterr()
-                mounted = False
-            finally:
-                try:
-                    if mounted:
-                        self.umount_l(d).terminate_geterr(fail_on_err = False)
-                    self.cleanup_mntpt()
-                except:
-                    logging.warn('stale mount possibly left behind on ' + d)
-            logging.debug('auxiliary glusterfs mount prepared')
-
-    class DirectMounter(Mounter):
-        """mounter backend which calls mount(8), umount(8) directly"""
-
-        mountkw = {'stderr': subprocess.PIPE}
-        glusterprog = 'glusterfs'
-
-        @staticmethod
-        def make_umount_argv(d):
-            return ['umount', '-l', d]
-
-        def make_mount_argv(self):
-            self.mntpt = tempfile.mkdtemp(prefix = 'gsyncd-aux-mount-')
-            return [self.get_glusterprog()] + ['--' + p for p in self.params] + [self.mntpt]
-
-        def cleanup_mntpt(self):
-            os.rmdir(self.mntpt)
-
-    class MountbrokerMounter(Mounter):
-        """mounter backend using the mountbroker gluster service"""
-
-        mountkw = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE}
-        glusterprog = 'gluster'
-
-        @classmethod
-        def make_cli_argv(cls):
-            return [cls.get_glusterprog()] + gconf.gluster_cli_options.split() + ['system::']
-
-        @classmethod
-        def make_umount_argv(cls, d):
-            return cls.make_cli_argv() + ['umount', d, 'lazy']
-
-        def make_mount_argv(self, label):
-            return self.make_cli_argv() + \
-                   ['mount', label, 'user-map-root=' + syncdutils.getusername()] + self.params
-
-        def handle_mounter(self, po):
-            self.mntpt = po.stdout.readline()[:-1]
-            po.stdout.close()
-            sup(self, po)
-            if po.returncode != 0:
-                # if cli terminated with error due to being
-                # refused by glusterd, what it put
-                # out on stdout is a diagnostic message
-                logging.error('glusterd answered: %s' % self.mntpt)
-
-    def connect(self):
-        """inhibit the resource beyond
-
-        Choose mounting backend (direct or mountbroker),
-        set up glusterfs parameters and perform the mount
-        with given backend
-        """
-
-        label = getattr(gconf, 'mountbroker', None)
-        if not label and not privileged():
-            label = syncdutils.getusername()
-        mounter = label and self.MountbrokerMounter or self.DirectMounter
-        params = gconf.gluster_params.split() + \
-                   (gconf.gluster_log_level and ['log-level=' + gconf.gluster_log_level] or []) + \
-                   ['log-file=' + gconf.gluster_log_file, 'volfile-server=' + self.host,
-                    'volfile-id=' + self.volume, 'client-pid=-1']
-        mounter(params).inhibit(*[l for l in [label] if l])
-
-    def connect_remote(self, *a, **kw):
-        sup(self, *a, **kw)
-        self.slavedir = "/proc/%d/cwd" % self.server.pid()
-
-    def service_loop(self, *args):
-        """enter service loop
-
-        - if slave given, instantiate GMaster and
-          pass control to that instance, which implements
-          master behavior
-        - else do that's what's inherited
-        """
-        if args:
-            GMaster(self, args[0]).crawl_loop()
-        else:
-            sup(self, *args)
-
-    def rsync(self, files):
-        return sup(self, files, self.slavedir)
-
-
-class SSH(AbstractUrl, SlaveRemote):
-    """scheme class for ssh:// urls
-
-    interface to remote slave on master side
-    implementing an ssh based proxy
-    """
-
-    def __init__(self, path):
-        self.remote_addr, inner_url = sup(self, path,
-                                          '^((?:%s@)?%s):(.+)' % tuple([ r.pattern for r in (UserRX, HostRX) ]))
-        self.inner_rsc = parse_url(inner_url)
-
-    def canonical_path(self):
-        m = re.match('([^@]+)@(.+)', self.remote_addr)
-        if m:
-            u, h = m.groups()
-        else:
-            u, h = syncdutils.getusername(), self.remote_addr
-        remote_addr = '@'.join([u, gethostbyname(h)])
-        return ':'.join([remote_addr, self.inner_rsc.get_url(canonical=True)])
-
-    def can_connect_to(self, remote):
-        """determine our position in the connectibility matrix"""
-        return False
-
-    def start_fd_client(self, *a, **opts):
-        """customizations for client startup
-
-        - be a no-op if we are to daemonize (client startup is deferred
-          to post-daemon stage)
-        - determine target url for rsync after consulting server
-        """
-        if opts.get('deferred'):
-            return a
-        sup(self, *a)
-        ityp = type(self.inner_rsc)
-        if ityp == FILE:
-            slavepath = self.inner_rsc.path
-        elif ityp == GLUSTER:
-            slavepath = "/proc/%d/cwd" % self.server.pid()
-        else:
-            raise NotImplementedError
-        self.slaveurl = ':'.join([self.remote_addr, slavepath])
-
-    def connect_remote(self, go_daemon=None):
-        """connect to inner slave url through outer ssh url
-
-        Wrap the connecting utility in ssh.
-
-        Much care is put into daemonizing: in that case
-        ssh is started before daemonization, but
-        RePCe client is to be created after that (as ssh
-        interactive password auth would be defeated by
-        a daemonized ssh, while client should be present
-        only in the final process). In that case the action
-        is taken apart to two parts, this method is ivoked
-        once pre-daemon, once post-daemon. Use @go_daemon
-        to deiced what part to perform.
-
-        [NB. ATM gluster product does not makes use of interactive
-        authentication.]
-        """
-        if go_daemon == 'done':
-            return self.start_fd_client(*self.fd_pair)
-        gconf.setup_ssh_ctl(tempfile.mkdtemp(prefix='gsyncd-aux-ssh-'))
-        deferred = go_daemon == 'postconn'
-        ret = sup(self, gconf.ssh_command.split() + gconf.ssh_ctl_args + [self.remote_addr], slave=self.inner_rsc.url, deferred=deferred)
-        if deferred:
-            # send a message to peer so that we can wait for
-            # the answer from which we know connection is
-            # established and we can proceed with daemonization
-            # (doing that too early robs the ssh passwd prompt...)
-            # However, we'd better not start the RepceClient
-            # before daemonization (that's not preserved properly
-            # in daemon), we just do a an ad-hoc linear put/get.
-            i, o = ret
-            inf = os.fdopen(i)
-            repce.send(o, None, '__repce_version__')
-            select((inf,), (), ())
-            repce.recv(inf)
-            # hack hack hack: store a global reference to the file
-            # to save it from getting GC'd which implies closing it
-            gconf.permanent_handles.append(inf)
-            self.fd_pair = (i, o)
-            return 'should'
-
-    def rsync(self, files):
-        return sup(self, files, '-ze', " ".join(gconf.ssh_command.split() + gconf.ssh_ctl_args), self.slaveurl)
diff --git a/xlators/features/marker/utils/syncdaemon/syncdutils.py b/xlators/features/marker/utils/syncdaemon/syncdutils.py
deleted file mode 100644
index 11c2063b7..000000000
--- a/xlators/features/marker/utils/syncdaemon/syncdutils.py
+++ /dev/null
@@ -1,273 +0,0 @@
-import os
-import sys
-import pwd
-import time
-import fcntl
-import shutil
-import logging
-from threading import Lock, Thread as baseThread
-from errno import EACCES, EAGAIN, EPIPE, ENOTCONN, EINTR
-from signal import SIGTERM, SIGKILL
-from time import sleep
-import select as oselect
-from os import waitpid as owaitpid
-try:
-    from cPickle import PickleError
-except ImportError:
-    # py 3
-    from pickle import PickleError
-
-from gconf import gconf
-
-try:
-    # py 3
-    from urllib import parse as urllib
-except ImportError:
-    import urllib
-
-def escape(s):
-    """the chosen flavor of string escaping, used all over
-       to turn whatever data to creatable representation"""
-    return urllib.quote_plus(s)
-
-def unescape(s):
-    """inverse of .escape"""
-    return urllib.unquote_plus(s)
-
-def norm(s):
-    if s:
-        return s.replace('-', '_')
-
-def update_file(path, updater, merger = lambda f: True):
-    """update a file in a transaction-like manner"""
-
-    fr = fw = None
-    try:
-        fd = os.open(path, os.O_CREAT|os.O_RDWR)
-        try:
-            fr = os.fdopen(fd, 'r+b')
-        except:
-            os.close(fd)
-            raise
-        fcntl.lockf(fr, fcntl.LOCK_EX)
-        if not merger(fr):
-            return
-
-        tmpp = path + '.tmp.' + str(os.getpid())
-        fd = os.open(tmpp, os.O_CREAT|os.O_EXCL|os.O_WRONLY)
-        try:
-            fw = os.fdopen(fd, 'wb', 0)
-        except:
-            os.close(fd)
-            raise
-        updater(fw)
-        os.fsync(fd)
-        os.rename(tmpp, path)
-    finally:
-        for fx in (fr, fw):
-            if fx:
-                fx.close()
-
-def grabfile(fname, content=None):
-    """open @fname + contest for its fcntl lock
-
-    @content: if given, set the file content to it
-    """
-    # damn those messy open() mode codes
-    fd = os.open(fname, os.O_CREAT|os.O_RDWR)
-    f = os.fdopen(fd, 'r+b', 0)
-    try:
-        fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB)
-    except:
-        ex = sys.exc_info()[1]
-        f.close()
-        if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN):
-            # cannot grab, it's taken
-            return
-        raise
-    if content:
-        try:
-            f.truncate()
-            f.write(content)
-        except:
-            f.close()
-            raise
-    gconf.permanent_handles.append(f)
-    return f
-
-def grabpidfile(fname=None, setpid=True):
-    """.grabfile customization for pid files"""
-    if not fname:
-        fname = gconf.pid_file
-    content = None
-    if setpid:
-        content = str(os.getpid()) + '\n'
-    return grabfile(fname, content=content)
-
-final_lock = Lock()
-
-def finalize(*a, **kw):
-    """all those messy final steps we go trough upon termination
-
-    Do away with pidfile, ssh control dir and logging.
-    """
-    final_lock.acquire()
-    if getattr(gconf, 'pid_file', None):
-        rm_pidf = gconf.pid_file_owned
-        if gconf.cpid:
-            # exit path from parent branch of daemonization
-            rm_pidf = False
-            while True:
-                f = grabpidfile(setpid=False)
-                if not f:
-                    # child has already taken over pidfile
-                    break
-                if os.waitpid(gconf.cpid, os.WNOHANG)[0] == gconf.cpid:
-                    # child has terminated
-                    rm_pidf = True
-                    break;
-                time.sleep(0.1)
-        if rm_pidf:
-            try:
-                os.unlink(gconf.pid_file)
-            except:
-                ex = sys.exc_info()[1]
-                if ex.errno == ENOENT:
-                    pass
-                else:
-                    raise
-    if gconf.ssh_ctl_dir and not gconf.cpid:
-        shutil.rmtree(gconf.ssh_ctl_dir)
-    if gconf.log_exit:
-        logging.info("exiting.")
-    sys.stdout.flush()
-    sys.stderr.flush()
-    os._exit(kw.get('exval', 0))
-
-def log_raise_exception(excont):
-    """top-level exception handler
-
-    Try to some fancy things to cover up we face with an error.
-    Translate some weird sounding but well understood exceptions
-    into human-friendly lingo
-    """
-    is_filelog = False
-    for h in logging.getLogger().handlers:
-        fno = getattr(getattr(h, 'stream', None), 'fileno', None)
-        if fno and not os.isatty(fno()):
-            is_filelog = True
-
-    exc = sys.exc_info()[1]
-    if isinstance(exc, SystemExit):
-        excont.exval = exc.code or 0
-        raise
-    else:
-        logtag = None
-        if isinstance(exc, GsyncdError):
-            if is_filelog:
-                logging.error(exc.message)
-            sys.stderr.write('failure: ' + exc.message + "\n")
-        elif isinstance(exc, PickleError) or isinstance(exc, EOFError) or \
-             ((isinstance(exc, OSError) or isinstance(exc, IOError)) and \
-              exc.errno == EPIPE):
-            logging.error('connection to peer is broken')
-            if hasattr(gconf, 'transport'):
-                gconf.transport.wait()
-                gconf.transport.terminate_geterr()
-        elif isinstance(exc, OSError) and exc.errno == ENOTCONN:
-            logging.error('glusterfs session went down')
-        else:
-            logtag = "FAIL"
-        if not logtag and logging.getLogger().isEnabledFor(logging.DEBUG):
-            logtag = "FULL EXCEPTION TRACE"
-        if logtag:
-            logging.exception(logtag + ": ")
-            sys.stderr.write("failed with %s.\n" % type(exc).__name__)
-        excont.exval = 1
-        sys.exit(excont.exval)
-
-
-class FreeObject(object):
-    """wildcard class for which any attribute can be set"""
-
-    def __init__(self, **kw):
-        for k,v in kw.items():
-            setattr(self, k, v)
-
-class Thread(baseThread):
-    """thread class flavor for gsyncd
-
-    - always a daemon thread
-    - force exit for whole program if thread
-      function coughs up an exception
-    """
-    def __init__(self, *a, **kw):
-        tf = kw.get('target')
-        if tf:
-            def twrap(*aa):
-                excont = FreeObject(exval = 0)
-                try:
-                    tf(*aa)
-                except:
-                    try:
-                        log_raise_exception(excont)
-                    finally:
-                        finalize(exval = excont.exval)
-            kw['target'] = twrap
-        baseThread.__init__(self, *a, **kw)
-        self.setDaemon(True)
-
-class GsyncdError(Exception):
-    pass
-
-def getusername(uid = None):
-    if uid == None:
-        uid = os.geteuid()
-    return pwd.getpwuid(uid).pw_name
-
-def privileged():
-    return os.geteuid() == 0
-
-def boolify(s):
-    """
-    Generic string to boolean converter
-
-    return
-    - Quick return if string 's' is of type bool
-    - True if it's in true_list
-    - False if it's in false_list
-    - Warn if it's not present in either and return False
-    """
-    true_list  = ['true', 'yes', '1', 'on']
-    false_list = ['false', 'no', '0', 'off']
-
-    if isinstance(s, bool):
-        return s
-
-    rv = False
-    lstr = s.lower()
-    if lstr in true_list:
-        rv = True
-    elif not lstr in false_list:
-        logging.warn("Unknown string (%s) in string to boolean conversion defaulting to False\n" % (s))
-
-    return rv
-
-def eintr_wrap(func, exc, *a):
-    """
-    wrapper around syscalls resilient to interrupt caused
-    by signals
-    """
-    while True:
-        try:
-            return func(*a)
-        except exc:
-            ex = sys.exc_info()[1]
-            if not ex.args[0] == EINTR:
-                raise GsyncdError(ex.args[1])
-
-def select(*a):
-    return eintr_wrap(oselect.select, oselect.error, *a)
-
-def waitpid (*a):
-    return eintr_wrap(owaitpid, OSError, *a)
diff --git a/xlators/features/path-convertor/src/Makefile.am b/xlators/features/path-convertor/src/Makefile.am
index 58cfed0f9..393a7bd08 100644
--- a/xlators/features/path-convertor/src/Makefile.am
+++ b/xlators/features/path-convertor/src/Makefile.am
@@ -2,13 +2,14 @@
 xlator_LTLIBRARIES = path-converter.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-path_converter_la_LDFLAGS = -module -avoidversion 
+path_converter_la_LDFLAGS = -module -avoid-version 
 
 path_converter_la_SOURCES = path.c
 path_converter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/path-convertor/src/path-mem-types.h b/xlators/features/path-convertor/src/path-mem-types.h
index c071513b6..77ada8d53 100644
--- a/xlators/features/path-convertor/src/path-mem-types.h
+++ b/xlators/features/path-convertor/src/path-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __PATH_MEM_TYPES_H__
 #define __PATH_MEM_TYPES_H__
 
diff --git a/xlators/features/path-convertor/src/path.c b/xlators/features/path-convertor/src/path.c
index f61630535..5c52e0a8d 100644
--- a/xlators/features/path-convertor/src/path.c
+++ b/xlators/features/path-convertor/src/path.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 /* TODO: add gf_log to all the cases returning errors */
 
 #ifndef _CONFIG_H
@@ -52,7 +42,7 @@ static char *
 name_this_to_that (xlator_t *xl, const char *path, const char *name)
 {
 	path_private_t *priv = xl->private;
-	char priv_path[ZR_PATH_MAX] = {0,};
+	char priv_path[PATH_MAX] = {0,};
 	char *tmp_name = NULL;
 	int32_t path_len = strlen (path);
 	int32_t name_len = strlen (name) - ZR_FILE_CONTENT_STRLEN;
@@ -848,8 +838,7 @@ path_setxattr (call_frame_t *frame,
 	if (tmp_path != loc_path)
 		GF_FREE (tmp_path);
 
-	if (tmp_name)
-		GF_FREE (tmp_name);
+	GF_FREE (tmp_name);
 
 	return 0;
 }
diff --git a/xlators/protocol/legacy/client/Makefile.am b/xlators/features/protect/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/protocol/legacy/client/Makefile.am
+++ b/xlators/features/protect/Makefile.am
diff --git a/xlators/features/protect/src/Makefile.am b/xlators/features/protect/src/Makefile.am
new file mode 100644
index 000000000..968e88c45
--- /dev/null
+++ b/xlators/features/protect/src/Makefile.am
@@ -0,0 +1,21 @@
+xlator_LTLIBRARIES = prot_dht.la prot_client.la prot_server.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+prot_dht_la_LDFLAGS = -module -avoid-version
+prot_dht_la_SOURCES = prot_dht.c
+prot_dht_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+prot_client_la_LDFLAGS = -module -avoid-version
+prot_client_la_SOURCES = prot_client.c
+prot_client_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+prot_server_la_LDFLAGS = -module -avoid-version
+prot_server_la_SOURCES = prot_server.c
+prot_server_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES = 
+
diff --git a/xlators/features/protect/src/prot_client.c b/xlators/features/protect/src/prot_client.c
new file mode 100644
index 000000000..d09715067
--- /dev/null
+++ b/xlators/features/protect/src/prot_client.c
@@ -0,0 +1,217 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+#ifndef __NetBSD__
+#include <execinfo.h>
+#endif
+
+#define NUM_FRAMES 20
+
+static char PROTECT_KEY[] = "trusted.glusterfs.protect";
+
+enum {
+        PROT_ACT_NONE = 0,
+        PROT_ACT_LOG,
+        PROT_ACT_REJECT,
+};
+
+void
+pcli_print_trace (char *name, call_frame_t *frame)
+{
+        void    *frames[NUM_FRAMES];
+        char    **symbols;
+        int     size;
+        int     i;
+
+        gf_log (name, GF_LOG_INFO, "Translator stack:");
+        while (frame) {
+                gf_log (name, GF_LOG_INFO, "%s (%s)",
+                        frame->wind_from, frame->this->name);
+                frame = frame->next;
+        }
+
+        size = backtrace(frames,NUM_FRAMES);
+        if (size <= 0) {
+                return;
+        }
+        symbols = backtrace_symbols(frames,size);
+        if (!symbols) {
+                return;
+        }
+
+        gf_log(name, GF_LOG_INFO, "Processor stack:");
+        for (i = 0; i < size; ++i) {
+                gf_log (name, GF_LOG_INFO, "%s", symbols[i]);
+        }
+        free(symbols);
+}
+
+int32_t
+pcli_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        uint64_t        value;
+
+        if (newloc->parent == oldloc->parent) {
+                gf_log (this->name, GF_LOG_DEBUG, "rename in same directory");
+                goto simple_unwind;
+        }
+        if (!oldloc->parent) {
+                goto simple_unwind;
+        }
+        if (inode_ctx_get(oldloc->parent,this,&value) != 0) {
+                goto simple_unwind;
+        }
+
+        if (value != PROT_ACT_NONE) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "got rename for protected %s", oldloc->path);
+                pcli_print_trace(this->name,frame->next);
+                if (value == PROT_ACT_REJECT) {
+                        STACK_UNWIND_STRICT (rename, frame, -1, EPERM,
+                                             NULL, NULL, NULL, NULL, NULL,
+                                             xdata);
+                        return 0;
+                }
+        }
+
+simple_unwind:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rename, oldloc, newloc,
+                         xdata);
+        return 0;
+}
+
+int32_t
+pcli_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+        data_t          *data;
+        uint64_t        value;
+
+        /*
+         * We can't use dict_get_str and strcmp here, because the value comes
+         * directly from the user and might not be NUL-terminated (it would
+         * be if we had set it ourselves.
+         */
+
+        data = dict_get(dict,PROTECT_KEY);
+        if (!data) {
+                goto simple_wind;
+        }
+
+        if (dict->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "attempted to mix %s with other keys", PROTECT_KEY);
+                goto simple_wind;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "got %s request", PROTECT_KEY);
+        if (!strncmp(data->data,"log",data->len)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "logging removals on %s", loc->path);
+                value = PROT_ACT_LOG;
+        }
+        else if (!strncmp(data->data,"reject",data->len)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "rejecting removals on %s", loc->path);
+                value = PROT_ACT_REJECT;
+        }
+        else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "removing protection on %s", loc->path);
+                value = PROT_ACT_NONE;
+        }
+        /* Right now the value doesn't matter - just the presence. */
+        if (inode_ctx_set(loc->inode,this,&value) != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set protection status for %s", loc->path);
+        }
+        STACK_UNWIND_STRICT (setxattr, frame, 0, 0, NULL);
+        return 0;
+
+simple_wind:
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                         loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+pcli_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
+{
+        uint64_t        value;
+
+        if (!loc->parent || (inode_ctx_get(loc->parent,this,&value) != 0)) {
+                goto simple_unwind;
+        }
+
+        if (value != PROT_ACT_NONE) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "got unlink for protected %s", loc->path);
+                pcli_print_trace(this->name,frame->next);
+                if (value == PROT_ACT_REJECT) {
+                        STACK_UNWIND_STRICT (unlink, frame, -1, EPERM,
+                                             NULL, NULL, NULL);
+                        return 0;
+                }
+        }
+
+simple_unwind:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+        .rename         = pcli_rename,
+        .setxattr       = pcli_setxattr,
+        .unlink         = pcli_unlink,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/protect/src/prot_dht.c b/xlators/features/protect/src/prot_dht.c
new file mode 100644
index 000000000..feec6ffd6
--- /dev/null
+++ b/xlators/features/protect/src/prot_dht.c
@@ -0,0 +1,168 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+enum gf_pdht_mem_types_ {
+        gf_pdht_mt_coord_t = gf_common_mt_end + 1,
+        gf_pdht_mt_end
+};
+
+typedef struct {
+        pthread_mutex_t         lock;
+        uint16_t                refs;
+        int32_t                 op_ret;
+        int32_t                 op_errno;
+        dict_t                  *xdata;
+} pdht_coord_t;
+
+static char PROTECT_KEY[] = "trusted.glusterfs.protect";
+
+void
+pdht_unref_and_unlock (call_frame_t *frame, xlator_t *this,
+                       pdht_coord_t *coord)
+{
+        gf_boolean_t    should_unwind;
+
+        should_unwind = (--(coord->refs) == 0);
+        pthread_mutex_unlock(&coord->lock);
+
+        if (should_unwind) {
+                STACK_UNWIND_STRICT (setxattr, frame, 
+                                     coord->op_ret, coord->op_errno,
+                                     coord->xdata);
+                if (coord->xdata) {
+                        dict_unref(coord->xdata);
+                }
+                GF_FREE(coord);
+        }
+}
+
+int32_t
+pdht_recurse_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        pdht_coord_t    *coord = cookie;
+
+        pthread_mutex_lock(&coord->lock);
+        if (op_ret) {
+                coord->op_ret = op_ret;
+                coord->op_errno = op_errno;
+        }
+        if (xdata) {
+                if (coord->xdata) {
+                        dict_unref(coord->xdata);
+                }
+                coord->xdata = dict_ref(xdata);
+        }
+        pdht_unref_and_unlock(frame,this,coord);
+                
+        return 0;
+}
+
+void
+pdht_recurse (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+              int32_t flags, dict_t *xdata, xlator_t *xl, pdht_coord_t *coord)
+{
+        xlator_list_t   *iter;
+
+        if (!strcmp(xl->type,"features/prot_client")) {
+                pthread_mutex_lock(&coord->lock);
+                ++(coord->refs);
+                pthread_mutex_unlock(&coord->lock);
+                STACK_WIND_COOKIE (frame, pdht_recurse_cbk, coord, xl,
+                                   xl->fops->setxattr, loc, dict, flags, xdata);
+        }
+
+        else for (iter = xl->children; iter; iter = iter->next) {
+                pdht_recurse (frame, this, loc, dict, flags, xdata,
+                              iter->xlator, coord);
+        }
+}
+
+int32_t
+pdht_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+        pdht_coord_t    *coord;
+
+        if (!dict_get(dict,PROTECT_KEY)) {
+                goto simple_wind;
+        }
+
+        if (dict->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "attempted to mix %s with other keys", PROTECT_KEY);
+                goto simple_wind;
+        }
+
+        coord = GF_CALLOC(1,sizeof(*coord),gf_pdht_mt_coord_t);
+        if (!coord) {
+                gf_log (this->name, GF_LOG_WARNING, "allocation failed");
+                goto simple_wind;
+        }
+
+        pthread_mutex_init(&coord->lock,NULL);
+        coord->refs = 1;
+        coord->op_ret = 0;
+        coord->xdata = NULL;
+
+        pdht_recurse(frame,this,loc,dict,flags,xdata,this,coord);
+        pthread_mutex_lock(&coord->lock);
+        pdht_unref_and_unlock(frame,this,coord);
+
+        return 0;
+
+simple_wind:
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                         loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+struct xlator_fops fops = {
+        .setxattr = pdht_setxattr,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/protect/src/prot_server.c b/xlators/features/protect/src/prot_server.c
new file mode 100644
index 000000000..beaee0889
--- /dev/null
+++ b/xlators/features/protect/src/prot_server.c
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/qemu-block/Makefile.am b/xlators/features/qemu-block/Makefile.am
new file mode 100644
index 000000000..af437a64d
--- /dev/null
+++ b/xlators/features/qemu-block/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src
diff --git a/xlators/features/qemu-block/src/Makefile.am b/xlators/features/qemu-block/src/Makefile.am
new file mode 100644
index 000000000..08a7b62a0
--- /dev/null
+++ b/xlators/features/qemu-block/src/Makefile.am
@@ -0,0 +1,155 @@
+if ENABLE_QEMU_BLOCK
+xlator_LTLIBRARIES = qemu-block.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+qemu_block_la_LDFLAGS = -module -avoid-version
+qemu_block_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GLIB_LIBS) -lz -lrt
+
+qemu_block_la_SOURCES_qemu = \
+	$(CONTRIBDIR)/qemu/qemu-coroutine.c \
+	$(CONTRIBDIR)/qemu/qemu-coroutine-lock.c \
+	$(CONTRIBDIR)/qemu/qemu-coroutine-sleep.c \
+	$(CONTRIBDIR)/qemu/coroutine-ucontext.c \
+	$(CONTRIBDIR)/qemu/block.c \
+	$(CONTRIBDIR)/qemu/nop-symbols.c
+
+qemu_block_la_SOURCES_qemu_util = \
+	$(CONTRIBDIR)/qemu/util/aes.c \
+	$(CONTRIBDIR)/qemu/util/bitmap.c \
+	$(CONTRIBDIR)/qemu/util/bitops.c \
+	$(CONTRIBDIR)/qemu/util/cutils.c \
+	$(CONTRIBDIR)/qemu/util/error.c \
+	$(CONTRIBDIR)/qemu/util/hbitmap.c \
+	$(CONTRIBDIR)/qemu/util/iov.c \
+	$(CONTRIBDIR)/qemu/util/module.c \
+	$(CONTRIBDIR)/qemu/util/oslib-posix.c \
+	$(CONTRIBDIR)/qemu/util/qemu-option.c \
+	$(CONTRIBDIR)/qemu/util/qemu-error.c \
+	$(CONTRIBDIR)/qemu/util/qemu-thread-posix.c \
+	$(CONTRIBDIR)/qemu/util/unicode.c \
+	$(CONTRIBDIR)/qemu/util/hexdump.c
+
+qemu_block_la_SOURCES_qemu_block = \
+	$(CONTRIBDIR)/qemu/block/snapshot.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-cache.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-cluster.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-refcount.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-snapshot.c \
+	$(CONTRIBDIR)/qemu/block/qcow2.c \
+	$(CONTRIBDIR)/qemu/block/qed-check.c \
+	$(CONTRIBDIR)/qemu/block/qed-cluster.c \
+	$(CONTRIBDIR)/qemu/block/qed-gencb.c \
+	$(CONTRIBDIR)/qemu/block/qed-l2-cache.c \
+	$(CONTRIBDIR)/qemu/block/qed-table.c \
+	$(CONTRIBDIR)/qemu/block/qed.c
+
+qemu_block_la_SOURCES_qemu_qobject = \
+	$(CONTRIBDIR)/qemu/qobject/json-lexer.c \
+	$(CONTRIBDIR)/qemu/qobject/json-parser.c \
+	$(CONTRIBDIR)/qemu/qobject/json-streamer.c \
+	$(CONTRIBDIR)/qemu/qobject/qbool.c \
+	$(CONTRIBDIR)/qemu/qobject/qdict.c \
+	$(CONTRIBDIR)/qemu/qobject/qerror.c \
+	$(CONTRIBDIR)/qemu/qobject/qfloat.c \
+	$(CONTRIBDIR)/qemu/qobject/qint.c \
+	$(CONTRIBDIR)/qemu/qobject/qjson.c \
+	$(CONTRIBDIR)/qemu/qobject/qlist.c \
+	$(CONTRIBDIR)/qemu/qobject/qstring.c
+
+qemu_block_la_SOURCES = \
+	$(qemu_block_la_SOURCES_qemu) \
+	$(qemu_block_la_SOURCES_qemu_util) \
+	$(qemu_block_la_SOURCES_qemu_block) \
+	$(qemu_block_la_SOURCES_qemu_qobject) \
+	bdrv-xlator.c \
+	coroutine-synctask.c \
+	bh-syncop.c \
+	monitor-logging.c \
+	clock-timer.c \
+	qemu-block.c \
+	qb-coroutines.c
+
+noinst_HEADERS_qemu = \
+	$(CONTRIBDIR)/qemu/config-host.h \
+	$(CONTRIBDIR)/qemu/qapi-types.h \
+	$(CONTRIBDIR)/qemu/qmp-commands.h \
+	$(CONTRIBDIR)/qemu/trace/generated-tracers.h \
+	$(CONTRIBDIR)/qemu/include/config.h \
+	$(CONTRIBDIR)/qemu/include/glib-compat.h \
+	$(CONTRIBDIR)/qemu/include/qemu-common.h \
+	$(CONTRIBDIR)/qemu/include/trace.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
+	$(CONTRIBDIR)/qemu/include/block/aio.h \
+	$(CONTRIBDIR)/qemu/include/block/block.h \
+	$(CONTRIBDIR)/qemu/include/block/block_int.h \
+	$(CONTRIBDIR)/qemu/include/block/blockjob.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine_int.h \
+	$(CONTRIBDIR)/qemu/include/block/snapshot.h \
+	$(CONTRIBDIR)/qemu/include/exec/cpu-common.h \
+	$(CONTRIBDIR)/qemu/include/exec/hwaddr.h \
+	$(CONTRIBDIR)/qemu/include/exec/poison.h \
+	$(CONTRIBDIR)/qemu/include/fpu/softfloat.h \
+	$(CONTRIBDIR)/qemu/include/migration/migration.h \
+	$(CONTRIBDIR)/qemu/include/migration/qemu-file.h \
+	$(CONTRIBDIR)/qemu/include/migration/vmstate.h \
+	$(CONTRIBDIR)/qemu/include/monitor/monitor.h \
+	$(CONTRIBDIR)/qemu/include/monitor/readline.h \
+	$(CONTRIBDIR)/qemu/include/qapi/error.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-lexer.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-parser.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-streamer.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qbool.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qdict.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qerror.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qfloat.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qint.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qjson.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qlist.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qobject.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qstring.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/types.h \
+	$(CONTRIBDIR)/qemu/include/qemu/aes.h \
+	$(CONTRIBDIR)/qemu/include/qemu/atomic.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bitmap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bitops.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bswap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/compiler.h \
+	$(CONTRIBDIR)/qemu/include/qemu/error-report.h \
+	$(CONTRIBDIR)/qemu/include/qemu/event_notifier.h \
+	$(CONTRIBDIR)/qemu/include/qemu/hbitmap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/host-utils.h \
+	$(CONTRIBDIR)/qemu/include/qemu/iov.h \
+	$(CONTRIBDIR)/qemu/include/qemu/main-loop.h \
+	$(CONTRIBDIR)/qemu/include/qemu/module.h \
+	$(CONTRIBDIR)/qemu/include/qemu/notify.h \
+	$(CONTRIBDIR)/qemu/include/qemu/option.h \
+	$(CONTRIBDIR)/qemu/include/qemu/option_int.h \
+	$(CONTRIBDIR)/qemu/include/qemu/osdep.h \
+	$(CONTRIBDIR)/qemu/include/qemu/queue.h \
+	$(CONTRIBDIR)/qemu/include/qemu/sockets.h \
+	$(CONTRIBDIR)/qemu/include/qemu/thread-posix.h \
+	$(CONTRIBDIR)/qemu/include/qemu/thread.h \
+	$(CONTRIBDIR)/qemu/include/qemu/timer.h \
+	$(CONTRIBDIR)/qemu/include/qemu/typedefs.h \
+	$(CONTRIBDIR)/qemu/include/sysemu/sysemu.h \
+	$(CONTRIBDIR)/qemu/include/sysemu/os-posix.h \
+	$(CONTRIBDIR)/qemu/block/qcow2.h \
+	$(CONTRIBDIR)/qemu/block/qed.h
+
+noinst_HEADERS = \
+	$(noinst_HEADERS_qemu) \
+	qemu-block.h \
+	qemu-block-memory-types.h \
+	qb-coroutines.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+            -I$(CONTRIBDIR)/qemu \
+            -I$(CONTRIBDIR)/qemu/include \
+            -DGLUSTER_XLATOR
+
+AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS) $(GLIB_CFLAGS)
+
+CLEANFILES =
+
+endif
diff --git a/xlators/features/qemu-block/src/bdrv-xlator.c b/xlators/features/qemu-block/src/bdrv-xlator.c
new file mode 100644
index 000000000..1e55b5fb7
--- /dev/null
+++ b/xlators/features/qemu-block/src/bdrv-xlator.c
@@ -0,0 +1,389 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "inode.h"
+#include "syncop.h"
+#include "qemu-block.h"
+#include "block/block_int.h"
+
+typedef struct BDRVGlusterState {
+	inode_t *inode;
+} BDRVGlusterState;
+
+static QemuOptsList runtime_opts = {
+	.name = "gluster",
+	.head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
+	.desc = {
+		{
+			.name = "filename",
+			.type = QEMU_OPT_STRING,
+			.help = "GFID of file",
+		},
+		{ /* end of list */ }
+	},
+};
+
+inode_t *
+qb_inode_from_filename (const char *filename)
+{
+	const char *iptr = NULL;
+	inode_t *inode = NULL;
+
+	iptr = filename + 17;
+	sscanf (iptr, "%p", &inode);
+
+	return inode;
+}
+
+
+int
+qb_inode_to_filename (inode_t *inode, char *filename, int size)
+{
+	return snprintf (filename, size, "gluster://inodep:%p", inode);
+}
+
+
+static fd_t *
+fd_from_bs (BlockDriverState *bs)
+{
+	BDRVGlusterState *s = bs->opaque;
+
+	return fd_anonymous (s->inode);
+}
+
+
+static int
+qemu_gluster_open (BlockDriverState *bs, QDict *options, int bdrv_flags)
+{
+	inode_t *inode = NULL;
+	BDRVGlusterState *s = bs->opaque;
+	QemuOpts *opts = NULL;
+	Error *local_err = NULL;
+	const char *filename = NULL;
+	char gfid_str[128];
+	int ret;
+	qb_conf_t *conf = THIS->private;
+
+	opts = qemu_opts_create_nofail(&runtime_opts);
+	qemu_opts_absorb_qdict(opts, options, &local_err);
+	if (error_is_set(&local_err)) {
+		qerror_report_err(local_err);
+		error_free(local_err);
+		return -EINVAL;
+	}
+
+	filename = qemu_opt_get(opts, "filename");
+
+	/*
+	 * gfid:<gfid> format means we're opening a backing image.
+	 */
+	ret = sscanf(filename, "gluster://gfid:%s", gfid_str);
+	if (ret) {
+		loc_t loc = {0,};
+		struct iatt buf = {0,};
+		uuid_t gfid;
+
+		uuid_parse(gfid_str, gfid);
+
+		loc.inode = inode_find(conf->root_inode->table, gfid);
+		if (!loc.inode) {
+			loc.inode = inode_new(conf->root_inode->table);
+			uuid_copy(loc.inode->gfid, gfid);
+		}
+
+		uuid_copy(loc.gfid, loc.inode->gfid);
+		ret = syncop_lookup(FIRST_CHILD(THIS), &loc, NULL, &buf, NULL,
+				    NULL);
+		if (ret) {
+			loc_wipe(&loc);
+			return ret;
+		}
+
+		s->inode = inode_ref(loc.inode);
+		loc_wipe(&loc);
+	} else {
+		inode = qb_inode_from_filename (filename);
+		if (!inode)
+			return -EINVAL;
+
+		s->inode = inode_ref(inode);
+	}
+
+	return 0;
+}
+
+
+static int
+qemu_gluster_create (const char *filename, QEMUOptionParameter *options)
+{
+	uint64_t total_size = 0;
+	inode_t *inode = NULL;
+	fd_t *fd = NULL;
+	struct iatt stat = {0, };
+	int ret = 0;
+
+	inode = qb_inode_from_filename (filename);
+	if (!inode)
+		return -EINVAL;
+
+	while (options && options->name) {
+		if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
+			total_size = options->value.n / BDRV_SECTOR_SIZE;
+		}
+		options++;
+	}
+
+	fd = fd_anonymous (inode);
+	if (!fd)
+		return -ENOMEM;
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &stat);
+	if (ret) {
+		fd_unref (fd);
+		return ret;
+	}
+
+	if (stat.ia_size) {
+		/* format ONLY if the filesize is 0 bytes */
+		fd_unref (fd);
+		return -EFBIG;
+	}
+
+	if (total_size) {
+		ret = syncop_ftruncate (FIRST_CHILD(THIS), fd, total_size);
+		if (ret) {
+			fd_unref (fd);
+			return ret;
+		}
+	}
+
+	fd_unref (fd);
+	return 0;
+}
+
+
+static int
+qemu_gluster_co_readv (BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+		       QEMUIOVector *qiov)
+{
+	fd_t *fd = NULL;
+	off_t offset = 0;
+	size_t size = 0;
+	struct iovec *iov = NULL;
+	int count = 0;
+	struct iobref *iobref = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+	if (!fd)
+		return -EIO;
+
+	offset = sector_num * BDRV_SECTOR_SIZE;
+	size = nb_sectors * BDRV_SECTOR_SIZE;
+
+	ret = syncop_readv (FIRST_CHILD(THIS), fd, size, offset, 0,
+			    &iov, &count, &iobref);
+	if (ret < 0)
+		goto out;
+
+	iov_copy (qiov->iov, qiov->niov, iov, count); /* *choke!* */
+
+out:
+	GF_FREE (iov);
+	if (iobref)
+		iobref_unref (iobref);
+	fd_unref (fd);
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_writev (BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+			QEMUIOVector *qiov)
+{
+	fd_t *fd = NULL;
+	off_t offset = 0;
+	size_t size = 0;
+	struct iobref *iobref = NULL;
+	struct iobuf *iobuf = NULL;
+	struct iovec iov = {0, };
+	int ret = -ENOMEM;
+
+	fd = fd_from_bs (bs);
+	if (!fd)
+		return -EIO;
+
+	offset = sector_num * BDRV_SECTOR_SIZE;
+	size = nb_sectors * BDRV_SECTOR_SIZE;
+
+	iobuf = iobuf_get2 (THIS->ctx->iobuf_pool, size);
+	if (!iobuf)
+		goto out;
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		goto out;
+	}
+
+	iobref_add (iobref, iobuf);
+
+	iov_unload (iobuf_ptr (iobuf), qiov->iov, qiov->niov); /* *choke!* */
+
+	iov.iov_base = iobuf_ptr (iobuf);
+	iov.iov_len = size;
+
+	ret = syncop_writev (FIRST_CHILD(THIS), fd, &iov, 1, offset, iobref, 0);
+
+out:
+	if (iobuf)
+		iobuf_unref (iobuf);
+	if (iobref)
+		iobref_unref (iobref);
+	fd_unref (fd);
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_flush (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_flush (FIRST_CHILD(THIS), fd);
+
+	fd_unref (fd);
+
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_fsync (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fsync (FIRST_CHILD(THIS), fd, 0);
+
+	fd_unref (fd);
+
+	return ret;
+}
+
+
+static int
+qemu_gluster_truncate (BlockDriverState *bs, int64_t offset)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_ftruncate (FIRST_CHILD(THIS), fd, offset);
+
+	fd_unref (fd);
+
+	return ret;
+}
+
+
+static int64_t
+qemu_gluster_getlength (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+	struct iatt iatt = {0, };
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &iatt);
+	if (ret < 0)
+		return -1;
+
+	return iatt.ia_size;
+}
+
+
+static int64_t
+qemu_gluster_allocated_file_size (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+	struct iatt iatt = {0, };
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &iatt);
+	if (ret < 0)
+		return -1;
+
+	return iatt.ia_blocks * 512;
+}
+
+
+static void
+qemu_gluster_close (BlockDriverState *bs)
+{
+	BDRVGlusterState *s = NULL;
+
+	s = bs->opaque;
+
+	inode_unref (s->inode);
+
+	return;
+}
+
+
+static QEMUOptionParameter qemu_gluster_create_options[] = {
+	{
+		.name = BLOCK_OPT_SIZE,
+		.type = OPT_SIZE,
+		.help = "Virtual disk size"
+	},
+	{ NULL }
+};
+
+
+static BlockDriver bdrv_gluster = {
+	.format_name                  = "gluster",
+	.protocol_name                = "gluster",
+	.instance_size                = sizeof(BDRVGlusterState),
+	.bdrv_file_open               = qemu_gluster_open,
+	.bdrv_close                   = qemu_gluster_close,
+	.bdrv_create                  = qemu_gluster_create,
+	.bdrv_getlength               = qemu_gluster_getlength,
+	.bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+	.bdrv_co_readv                = qemu_gluster_co_readv,
+	.bdrv_co_writev               = qemu_gluster_co_writev,
+	.bdrv_co_flush_to_os          = qemu_gluster_co_flush,
+	.bdrv_co_flush_to_disk        = qemu_gluster_co_fsync,
+	.bdrv_truncate                = qemu_gluster_truncate,
+	.create_options               = qemu_gluster_create_options,
+};
+
+
+static void bdrv_gluster_init(void)
+{
+	bdrv_register(&bdrv_gluster);
+}
+
+
+block_init(bdrv_gluster_init);
diff --git a/xlators/features/qemu-block/src/bh-syncop.c b/xlators/features/qemu-block/src/bh-syncop.c
new file mode 100644
index 000000000..e8686f6d4
--- /dev/null
+++ b/xlators/features/qemu-block/src/bh-syncop.c
@@ -0,0 +1,48 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "block/aio.h"
+
+void
+qemu_bh_schedule (QEMUBH *bh)
+{
+	return;
+}
+
+void
+qemu_bh_cancel (QEMUBH *bh)
+{
+	return;
+}
+
+void
+qemu_bh_delete (QEMUBH *bh)
+{
+
+}
+
+QEMUBH *
+qemu_bh_new (QEMUBHFunc *cb, void *opaque)
+{
+	return NULL;
+}
diff --git a/xlators/features/qemu-block/src/clock-timer.c b/xlators/features/qemu-block/src/clock-timer.c
new file mode 100644
index 000000000..fcbec6ad1
--- /dev/null
+++ b/xlators/features/qemu-block/src/clock-timer.c
@@ -0,0 +1,60 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "qemu/timer.h"
+
+QEMUClock *vm_clock;
+int use_rt_clock = 0;
+
+QEMUTimer *qemu_new_timer (QEMUClock *clock, int scale,
+			   QEMUTimerCB *cb, void *opaque)
+{
+	return NULL;
+}
+
+int64_t qemu_get_clock_ns (QEMUClock *clock)
+{
+	return 0;
+}
+
+void qemu_mod_timer (QEMUTimer *ts, int64_t expire_time)
+{
+	return;
+}
+
+void qemu_free_timer (QEMUTimer *ts)
+{
+
+}
+
+void qemu_del_timer (QEMUTimer *ts)
+{
+
+}
+
+bool qemu_aio_wait()
+{
+	synctask_wake (synctask_get());
+	synctask_yield (synctask_get());
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/coroutine-synctask.c b/xlators/features/qemu-block/src/coroutine-synctask.c
new file mode 100644
index 000000000..e43988a95
--- /dev/null
+++ b/xlators/features/qemu-block/src/coroutine-synctask.c
@@ -0,0 +1,116 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "qemu-block.h"
+
+/*
+ * This code serves as the bridge from the main glusterfs context to the qemu
+ * coroutine context via synctask. We create a single threaded syncenv with a
+ * single synctask responsible for processing a queue of coroutines. The qemu
+ * code invoked from within the synctask function handlers uses the ucontext
+ * coroutine implementation and scheduling logic internal to qemu. This
+ * effectively donates a thread of execution to qemu and its internal coroutine
+ * management.
+ *
+ * NOTE: The existence of concurrent synctasks has proven quite racy with regard
+ * to qemu coroutine management, particularly related to the lifecycle
+ * differences with top-level synctasks and internally created coroutines and
+ * interactions with qemu-internal queues (and locks, in turn). We explicitly
+ * disallow this scenario, via the queue, until it is more well supported.
+ */
+
+static struct {
+	struct list_head	queue;
+	gf_lock_t		lock;
+	struct synctask		*task;
+} qb_co;
+
+static void
+init_qbco()
+{
+	INIT_LIST_HEAD(&qb_co.queue);
+	LOCK_INIT(&qb_co.lock);
+}
+
+static int
+synctask_nop_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+	return 0;
+}
+
+static int
+qb_synctask_wrap (void *opaque)
+{
+	qb_local_t *qb_local, *tmp;
+
+	LOCK(&qb_co.lock);
+
+	while (!list_empty(&qb_co.queue)) {
+		list_for_each_entry_safe(qb_local, tmp, &qb_co.queue, list) {
+			list_del_init(&qb_local->list);
+			break;
+		}
+
+		UNLOCK(&qb_co.lock);
+
+		qb_local->synctask_fn(qb_local);
+		/* qb_local is now unwound and gone! */
+
+		LOCK(&qb_co.lock);
+	}
+
+	qb_co.task = NULL;
+
+	UNLOCK(&qb_co.lock);
+
+	return 0;
+}
+
+int
+qb_coroutine (call_frame_t *frame, synctask_fn_t fn)
+{
+	qb_local_t *qb_local = NULL;
+	qb_conf_t *qb_conf = NULL;
+	static int init = 0;
+
+	qb_local = frame->local;
+	qb_local->synctask_fn = fn;
+	qb_conf = frame->this->private;
+
+	if (!init) {
+		init = 1;
+		init_qbco();
+	}
+
+	LOCK(&qb_co.lock);
+
+	if (!qb_co.task)
+		qb_co.task = synctask_create(qb_conf->env, qb_synctask_wrap,
+					     synctask_nop_cbk, frame, NULL);
+
+	list_add_tail(&qb_local->list, &qb_co.queue);
+
+	UNLOCK(&qb_co.lock);
+
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/monitor-logging.c b/xlators/features/qemu-block/src/monitor-logging.c
new file mode 100644
index 000000000..d37c37f0f
--- /dev/null
+++ b/xlators/features/qemu-block/src/monitor-logging.c
@@ -0,0 +1,50 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "qemu-block-memory-types.h"
+
+#include "block/block_int.h"
+
+Monitor *cur_mon;
+
+int
+monitor_cur_is_qmp()
+{
+	/* No QMP support here */
+	return 0;
+}
+
+void
+monitor_set_error (Monitor *mon, QError *qerror)
+{
+	/* NOP here */
+	return;
+}
+
+
+void
+monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
+{
+	char buf[4096];
+
+	vsnprintf(buf, sizeof(buf), fmt, ap);
+
+	gf_log (THIS->name, GF_LOG_ERROR, "%s", buf);
+}
diff --git a/xlators/features/qemu-block/src/qb-coroutines.c b/xlators/features/qemu-block/src/qb-coroutines.c
new file mode 100644
index 000000000..974312f12
--- /dev/null
+++ b/xlators/features/qemu-block/src/qb-coroutines.c
@@ -0,0 +1,667 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "inode.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "qemu-block-memory-types.h"
+#include "qemu-block.h"
+#include "qb-coroutines.h"
+
+
+int
+qb_format_and_resume (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	char filename[64];
+	char base_filename[128];
+	int use_base = 0;
+	qb_inode_t *qb_inode = NULL;
+	Error *local_err = NULL;
+	fd_t *fd = NULL;
+	dict_t *xattr = NULL;
+	qb_conf_t *qb_conf = NULL;
+	int ret = -1;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+	qb_conf = frame->this->private;
+
+	qb_inode_to_filename (inode, filename, 64);
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+
+	/*
+	 * See if the caller specified a backing image.
+	 */
+	if (!uuid_is_null(qb_inode->backing_gfid) || qb_inode->backing_fname) {
+		loc_t loc = {0,};
+		char gfid_str[64];
+		struct iatt buf;
+
+		if (!uuid_is_null(qb_inode->backing_gfid)) {
+			loc.inode = inode_find(qb_conf->root_inode->table,
+					qb_inode->backing_gfid);
+			if (!loc.inode) {
+				loc.inode = inode_new(qb_conf->root_inode->table);
+				uuid_copy(loc.inode->gfid,
+					qb_inode->backing_gfid);
+			}
+			uuid_copy(loc.gfid, loc.inode->gfid);
+		} else if (qb_inode->backing_fname) {
+			loc.inode = inode_new(qb_conf->root_inode->table);
+			loc.name = qb_inode->backing_fname;
+			loc.parent = inode_parent(inode, NULL, NULL);
+			loc_path(&loc, loc.name);
+		}
+
+		/*
+		 * Lookup the backing image. Verify existence and/or get the
+		 * gfid if we don't already have it.
+		 */
+		ret = syncop_lookup(FIRST_CHILD(frame->this), &loc, NULL, &buf,
+				    NULL, NULL);
+		GF_FREE(qb_inode->backing_fname);
+		if (ret) {
+			loc_wipe(&loc);
+			ret = -ret;
+			goto err;
+		}
+
+		uuid_copy(qb_inode->backing_gfid, buf.ia_gfid);
+		loc_wipe(&loc);
+
+		/*
+		 * We pass the filename of the backing image into the qemu block
+		 * subsystem as the associated gfid. This is embedded into the
+		 * clone image and passed along to the gluster bdrv backend when
+		 * the block subsystem needs to operate on the backing image on
+		 * behalf of the clone.
+		 */
+		uuid_unparse(qb_inode->backing_gfid, gfid_str);
+		snprintf(base_filename, sizeof(base_filename),
+			 "gluster://gfid:%s", gfid_str);
+		use_base = 1;
+	}
+
+	bdrv_img_create (filename, qb_inode->fmt,
+			 use_base ? base_filename : NULL, 0, 0, qb_inode->size,
+			 0, &local_err, true);
+
+	if (error_is_set (&local_err)) {
+		gf_log (frame->this->name, GF_LOG_ERROR, "%s",
+			error_get_pretty (local_err));
+		error_free (local_err);
+		QB_STUB_UNWIND (stub, -1, EIO);
+		return 0;
+	}
+
+	fd = fd_anonymous (inode);
+	if (!fd) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not create anonymous fd for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	xattr = dict_new ();
+	if (!xattr) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not allocate xattr dict for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		fd_unref (fd);
+		return 0;
+	}
+
+	ret = dict_set_str (xattr, qb_conf->qb_xattr_key, local->fmt);
+	if (ret) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not dict_set for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		fd_unref (fd);
+		dict_unref (xattr);
+		return 0;
+	}
+
+	ret = syncop_fsetxattr (FIRST_CHILD(THIS), fd, xattr, 0);
+	if (ret) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"failed to setxattr for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, -ret);
+		fd_unref (fd);
+		dict_unref (xattr);
+		return 0;
+	}
+
+	fd_unref (fd);
+	dict_unref (xattr);
+
+	QB_STUB_UNWIND (stub, 0, 0);
+
+	return 0;
+
+err:
+	QB_STUB_UNWIND(stub, -1, ret);
+	return 0;
+}
+
+
+static BlockDriverState *
+qb_bs_create (inode_t *inode, const char *fmt)
+{
+	char filename[64];
+	BlockDriverState *bs = NULL;
+	BlockDriver *drv = NULL;
+	int op_errno = 0;
+	int ret = 0;
+
+	bs = bdrv_new (uuid_utoa (inode->gfid));
+	if (!bs) {
+		op_errno = ENOMEM;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"could not allocate @bdrv for gfid:%s",
+			uuid_utoa (inode->gfid));
+		goto err;
+	}
+
+	drv = bdrv_find_format (fmt);
+	if (!drv) {
+		op_errno = EINVAL;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"Unknown file format: %s for gfid:%s",
+			fmt, uuid_utoa (inode->gfid));
+		goto err;
+	}
+
+	qb_inode_to_filename (inode, filename, 64);
+
+	ret = bdrv_open (bs, filename, NULL, BDRV_O_RDWR, drv);
+	if (ret < 0) {
+		op_errno = -ret;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"Unable to bdrv_open() gfid:%s (%s)",
+			uuid_utoa (inode->gfid), strerror (op_errno));
+		goto err;
+	}
+
+	return bs;
+err:
+	errno = op_errno;
+	return NULL;
+}
+
+
+int
+qb_co_open (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+	qb_inode->refcnt++;
+
+	QB_STUB_RESUME (stub);
+
+	return 0;
+}
+
+
+int
+qb_co_writev (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	QEMUIOVector qiov = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	qemu_iovec_init_external (&qiov, stub->args.vector, stub->args.count);
+
+	ret = bdrv_pwritev (qb_inode->bs, stub->args.offset, &qiov);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_co_readv (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	struct iobuf *iobuf = NULL;
+	struct iobref *iobref = NULL;
+	struct iovec iov = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	if (stub->args.offset >= qb_inode->size) {
+		QB_STUB_UNWIND (stub, 0, 0);
+		return 0;
+	}
+
+	iobuf = iobuf_get2 (frame->this->ctx->iobuf_pool, stub->args.size);
+	if (!iobuf) {
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		iobuf_unref (iobuf);
+		return 0;
+	}
+
+	if (iobref_add (iobref, iobuf) < 0) {
+		iobuf_unref (iobuf);
+		iobref_unref (iobref);
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	ret = bdrv_pread (qb_inode->bs, stub->args.offset, iobuf_ptr (iobuf),
+			  stub->args.size);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+		iobref_unref (iobref);
+		return 0;
+	}
+
+	iov.iov_base = iobuf_ptr (iobuf);
+	iov.iov_len = ret;
+
+	stub->args_cbk.vector = iov_dup (&iov, 1);
+	stub->args_cbk.count = 1;
+	stub->args_cbk.iobref = iobref;
+
+	QB_STUB_UNWIND (stub, ret, 0);
+
+	return 0;
+}
+
+
+int
+qb_co_fsync (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_flush (qb_inode->bs);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+static void
+qb_update_size_xattr (xlator_t *this, fd_t *fd, const char *fmt, off_t offset)
+{
+	char val[QB_XATTR_VAL_MAX];
+	qb_conf_t *qb_conf = NULL;
+	dict_t *xattr = NULL;
+
+	qb_conf = this->private;
+
+	snprintf (val, QB_XATTR_VAL_MAX, "%s:%llu",
+		  fmt, (long long unsigned) offset);
+
+	xattr = dict_new ();
+	if (!xattr)
+		return;
+
+	if (dict_set_str (xattr, qb_conf->qb_xattr_key, val) != 0) {
+		dict_unref (xattr);
+		return;
+	}
+
+	syncop_fsetxattr (FIRST_CHILD(this), fd, xattr, 0);
+	dict_unref (xattr);
+}
+
+
+int
+qb_co_truncate (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+	off_t offset = 0;
+	xlator_t *this = NULL;
+
+	this = THIS;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = syncop_fstat (FIRST_CHILD(this), local->fd,
+                            &stub->args_cbk.prestat);
+        if (ret < 0)
+                goto out;
+	stub->args_cbk.prestat.ia_size = qb_inode->size;
+
+	ret = bdrv_truncate (qb_inode->bs, stub->args.offset);
+	if (ret < 0)
+		goto out;
+
+	offset = bdrv_getlength (qb_inode->bs);
+
+	qb_inode->size = offset;
+
+	ret = syncop_fstat (FIRST_CHILD(this), local->fd,
+                            &stub->args_cbk.poststat);
+        if (ret < 0)
+                goto out;
+	stub->args_cbk.poststat.ia_size = qb_inode->size;
+
+	qb_update_size_xattr (this, local->fd, qb_inode->fmt, qb_inode->size);
+
+out:
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_co_close (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	BlockDriverState *bs = NULL;
+
+	local = opaque;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (THIS, inode);
+
+	if (!--qb_inode->refcnt) {
+		bs = qb_inode->bs;
+		qb_inode->bs = NULL;
+		bdrv_delete (bs);
+	}
+
+	frame = local->frame;
+	frame->local = NULL;
+	qb_local_free (THIS, local);
+	STACK_DESTROY (frame->root);
+
+	return 0;
+}
+
+
+int
+qb_snapshot_create (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	QEMUSnapshotInfo sn;
+	struct timeval tv = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	memset (&sn, 0, sizeof (sn));
+        pstrcpy (sn.name, sizeof(sn.name), local->name);
+        gettimeofday (&tv, NULL);
+        sn.date_sec = tv.tv_sec;
+        sn.date_nsec = tv.tv_usec * 1000;
+
+        ret = bdrv_snapshot_create (qb_inode->bs, &sn);
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_snapshot_delete (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_snapshot_delete (qb_inode->bs, local->name);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_snapshot_goto (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_snapshot_goto (qb_inode->bs, local->name);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/qb-coroutines.h b/xlators/features/qemu-block/src/qb-coroutines.h
new file mode 100644
index 000000000..583319f3b
--- /dev/null
+++ b/xlators/features/qemu-block/src/qb-coroutines.h
@@ -0,0 +1,30 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QB_COROUTINES_H
+#define __QB_COROUTINES_H
+
+#include "syncop.h"
+#include "call-stub.h"
+#include "block/block_int.h"
+#include "monitor/monitor.h"
+
+int qb_format_and_resume (void *opaque);
+int qb_snapshot_create (void *opaque);
+int qb_snapshot_delete (void *opaque);
+int qb_snapshot_goto (void *opaque);
+int qb_co_open (void *opaque);
+int qb_co_close (void *opaque);
+int qb_co_writev (void *opaque);
+int qb_co_readv (void *opaque);
+int qb_co_fsync (void *opaque);
+int qb_co_truncate (void *opaque);
+
+#endif /* __QB_COROUTINES_H */
diff --git a/xlators/features/qemu-block/src/qemu-block-memory-types.h b/xlators/features/qemu-block/src/qemu-block-memory-types.h
new file mode 100644
index 000000000..267b3893f
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block-memory-types.h
@@ -0,0 +1,25 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __QB_MEM_TYPES_H__
+#define __QB_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_qb_mem_types_ {
+        gf_qb_mt_qb_conf_t   = gf_common_mt_end + 1,
+        gf_qb_mt_qb_inode_t,
+        gf_qb_mt_qb_local_t,
+	gf_qb_mt_coroutinesynctask_t,
+        gf_qb_mt_end
+};
+#endif
+
diff --git a/xlators/features/qemu-block/src/qemu-block.c b/xlators/features/qemu-block/src/qemu-block.c
new file mode 100644
index 000000000..48bbf3140
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block.c
@@ -0,0 +1,1140 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "inode.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "qemu-block-memory-types.h"
+#include "qemu-block.h"
+#include "qb-coroutines.h"
+
+
+qb_inode_t *
+__qb_inode_ctx_get (xlator_t *this, inode_t *inode)
+{
+        uint64_t    value    = 0;
+        qb_inode_t *qb_inode = NULL;
+
+        __inode_ctx_get (inode, this, &value);
+        qb_inode = (qb_inode_t *)(unsigned long) value;
+
+        return qb_inode;
+}
+
+
+qb_inode_t *
+qb_inode_ctx_get (xlator_t *this, inode_t *inode)
+{
+        qb_inode_t *qb_inode = NULL;
+
+        LOCK (&inode->lock);
+        {
+                qb_inode = __qb_inode_ctx_get (this, inode);
+        }
+        UNLOCK (&inode->lock);
+
+        return qb_inode;
+}
+
+
+qb_inode_t *
+qb_inode_ctx_del (xlator_t *this, inode_t *inode)
+{
+        uint64_t    value    = 0;
+        qb_inode_t *qb_inode = NULL;
+
+        inode_ctx_del (inode, this, &value);
+        qb_inode = (qb_inode_t *)(unsigned long) value;
+
+        return qb_inode;
+}
+
+
+int
+qb_inode_cleanup (xlator_t *this, inode_t *inode, int warn)
+{
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_del (this, inode);
+
+	if (!qb_inode)
+		return 0;
+
+	if (warn)
+		gf_log (this->name, GF_LOG_WARNING,
+			"inode %s no longer block formatted",
+			uuid_utoa (inode->gfid));
+
+	/* free (qb_inode->bs); */
+
+	GF_FREE (qb_inode);
+
+	return 0;
+}
+
+
+int
+qb_iatt_fixup (xlator_t *this, inode_t *inode, struct iatt *iatt)
+{
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, inode);
+	if (!qb_inode)
+		return 0;
+
+	iatt->ia_size = qb_inode->size;
+
+	return 0;
+}
+
+
+int
+qb_format_extract (xlator_t *this, char *format, inode_t *inode)
+{
+	char       *s, *save;
+	uint64_t    size = 0;
+	char        fmt[QB_XATTR_VAL_MAX+1] = {0, };
+	qb_inode_t *qb_inode = NULL;
+	char *formatstr = NULL;
+	uuid_t gfid = {0,};
+	char gfid_str[64] = {0,};
+	int ret;
+
+	strncpy(fmt, format, QB_XATTR_VAL_MAX);
+
+	s = strtok_r(fmt, ":", &save);
+	if (!s)
+		goto invalid;
+	formatstr = gf_strdup(s);
+
+	s = strtok_r(NULL, ":", &save);
+	if (!s)
+		goto invalid;
+	if (gf_string2bytesize (s, &size))
+		goto invalid;
+	if (!size)
+		goto invalid;
+
+	s = strtok_r(NULL, "\0", &save);
+	if (s && !strncmp(s, "<gfid:", strlen("<gfid:"))) {
+		/*
+		 * Check for valid gfid backing image specifier.
+		 */
+		if (strlen(s) + 1 > sizeof(gfid_str))
+			goto invalid;
+		ret = sscanf(s, "<gfid:%[^>]s", gfid_str);
+		if (ret == 1) {
+			ret = uuid_parse(gfid_str, gfid);
+			if (ret < 0)
+				goto invalid;
+		}
+	}
+
+	qb_inode = qb_inode_ctx_get (this, inode);
+	if (!qb_inode)
+		qb_inode = GF_CALLOC (1, sizeof (*qb_inode),
+				      gf_qb_mt_qb_inode_t);
+	if (!qb_inode) {
+		GF_FREE(formatstr);
+		return ENOMEM;
+	}
+
+	strncpy(qb_inode->fmt, formatstr, QB_XATTR_VAL_MAX);
+	qb_inode->size = size;
+
+	/*
+	 * If a backing gfid was not specified, interpret any remaining bytes
+	 * associated with a backing image as a filename local to the parent
+	 * directory. The format processing will validate further.
+	 */
+	if (!uuid_is_null(gfid))
+		uuid_copy(qb_inode->backing_gfid, gfid);
+	else if (s)
+		qb_inode->backing_fname = gf_strdup(s);
+
+	inode_ctx_set (inode, this, (void *)&qb_inode);
+
+	GF_FREE(formatstr);
+
+	return 0;
+
+invalid:
+	GF_FREE(formatstr);
+
+	gf_log (this->name, GF_LOG_WARNING,
+		"invalid format '%s' in inode %s", format,
+		uuid_utoa (inode->gfid));
+	return EINVAL;
+}
+
+
+void
+qb_local_free (xlator_t *this, qb_local_t *local)
+{
+	if (local->inode)
+		inode_unref (local->inode);
+	if (local->fd)
+		fd_unref (local->fd);
+	GF_FREE (local);
+}
+
+
+int
+qb_local_init (call_frame_t *frame)
+{
+	qb_local_t *qb_local = NULL;
+
+	qb_local = GF_CALLOC (1, sizeof (*qb_local), gf_qb_mt_qb_local_t);
+	if (!qb_local)
+		return -1;
+	INIT_LIST_HEAD(&qb_local->list);
+
+	qb_local->frame = frame;
+	frame->local = qb_local;
+
+	return 0;
+}
+
+
+int
+qb_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	       int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+	       dict_t *xdata, struct iatt *postparent)
+{
+	char *format = NULL;
+	qb_conf_t *conf = NULL;
+
+	conf = this->private;
+
+	if (op_ret == -1)
+		goto out;
+
+	/*
+	 * Cache the root inode for dealing with backing images. The format
+	 * coroutine and the gluster qemu backend driver both use the root inode
+	 * table to verify and/or redirect I/O to the backing image via
+	 * anonymous fd's.
+	 */
+	if (!conf->root_inode && __is_root_gfid(inode->gfid))
+		conf->root_inode = inode_ref(inode);
+
+	if (!xdata)
+		goto out;
+
+	if (dict_get_str (xdata, conf->qb_xattr_key, &format))
+		goto out;
+
+	if (!format) {
+		qb_inode_cleanup (this, inode, 1);
+		goto out;
+	}
+
+	op_errno = qb_format_extract (this, format, inode);
+	if (op_errno)
+		op_ret = -1;
+
+	qb_iatt_fixup (this, inode, buf);
+out:
+	QB_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+			 xdata, postparent);
+	return 0;
+}
+
+
+int
+qb_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+	qb_conf_t *conf = NULL;
+
+	conf = this->private;
+
+	xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+	if (!xdata)
+		goto enomem;
+
+	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
+		goto enomem;
+
+	STACK_WIND (frame, qb_lookup_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+	dict_unref (xdata);
+	return 0;
+enomem:
+	QB_STACK_UNWIND (lookup, frame, -1, ENOMEM, 0, 0, 0, 0);
+	if (xdata)
+		dict_unref (xdata);
+	return 0;
+}
+
+
+int
+qb_setxattr_format (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
+		    dict_t *xattr, inode_t *inode)
+{
+	char *format = NULL;
+	int op_errno = 0;
+	qb_local_t *qb_local = NULL;
+	data_t *data = NULL;
+	qb_inode_t *qb_inode;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-format"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	format = alloca (data->len + 1);
+	memcpy (format, data->data, data->len);
+	format[data->len] = 0;
+
+	op_errno = qb_format_extract (this, format, inode);
+	if (op_errno) {
+		QB_STUB_UNWIND (stub, -1, op_errno);
+		return 0;
+	}
+	qb_inode = qb_inode_ctx_get(this, inode);
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+
+	snprintf(qb_local->fmt, QB_XATTR_VAL_MAX, "%s:%lu", qb_inode->fmt,
+		 qb_inode->size);
+
+	qb_coroutine (frame, qb_format_and_resume);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_snapshot_create (call_frame_t *frame, xlator_t *this,
+			     call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-create"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_create);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_snapshot_delete (call_frame_t *frame, xlator_t *this,
+			     call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-delete"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_delete);
+
+	return 0;
+}
+
+int
+qb_setxattr_snapshot_goto (call_frame_t *frame, xlator_t *this,
+			   call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-goto"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_goto);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_common (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
+		    dict_t *xattr, inode_t *inode)
+{
+	data_t *data = NULL;
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-format"))) {
+		qb_setxattr_format (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-create"))) {
+		qb_setxattr_snapshot_create (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-delete"))) {
+		qb_setxattr_snapshot_delete (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-goto"))) {
+		qb_setxattr_snapshot_goto (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	QB_STUB_RESUME (stub);
+
+	return 0;
+}
+
+
+int
+qb_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
+	     int flags, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	stub = fop_setxattr_stub (frame, default_setxattr_resume, loc, xattr,
+				  flags, xdata);
+	if (!stub)
+		goto enomem;
+
+	qb_setxattr_common (frame, this, stub, xattr, loc->inode);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (setxattr, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+
+int
+qb_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+	      int flags, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume, fd, xattr,
+				   flags, xdata);
+	if (!stub)
+		goto enomem;
+
+	qb_setxattr_common (frame, this, stub, xattr, fd->inode);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (fsetxattr, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+
+int
+qb_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	     int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+	qb_local_t *qb_local = NULL;
+
+	qb_local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	if (!qb_inode_ctx_get (this, qb_local->inode))
+		goto unwind;
+
+	stub = fop_open_cbk_stub (frame, NULL, op_ret, op_errno, fd, xdata);
+	if (!stub) {
+		op_ret = -1;
+		op_errno = ENOMEM;
+		goto unwind;
+	}
+
+	qb_local->stub = stub;
+
+	qb_coroutine (frame, qb_co_open);
+
+	return 0;
+unwind:
+	QB_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
+	return 0;
+}
+
+
+int
+qb_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	 fd_t *fd, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, loc->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->open, loc, flags, fd,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (loc->inode);
+	qb_local->fd = fd_ref (fd);
+
+	STACK_WIND (frame, qb_open_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+	return 0;
+enomem:
+	QB_STACK_UNWIND (open, frame, -1, ENOMEM, 0, 0);
+	return 0;
+}
+
+
+int
+qb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
+	   dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->writev, fd, vector, count,
+			    offset, flags, iobref, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_writev_stub (frame, NULL, fd, vector, count,
+					  offset, flags, iobref, xdata);
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_writev);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (writev, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	  off_t offset, uint32_t flags, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->readv, fd, size, offset,
+			    flags, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_readv_stub (frame, NULL, fd, size, offset,
+					  flags, xdata);
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_readv);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (readv, frame, -1, ENOMEM, 0, 0, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int dsync,
+	  dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->fsync, fd, dsync, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_fsync_stub (frame, NULL, fd, dsync, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_fsync);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->flush, fd, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_flush_stub (frame, NULL, fd, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_fsync);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (flush, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+static int32_t
+qb_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		dict_t *xdata)
+{
+	qb_conf_t *conf = this->private;
+	gf_dirent_t *entry;
+	char *format;
+
+	list_for_each_entry(entry, &entries->list, list) {
+		if (!entry->inode || !entry->dict)
+			continue;
+
+		format = NULL;
+		if (dict_get_str(entry->dict, conf->qb_xattr_key, &format))
+			continue;
+
+		if (!format) {
+			qb_inode_cleanup(this, entry->inode, 1);
+			continue;
+		}
+
+		if (qb_format_extract(this, format, entry->inode))
+			continue;
+
+		qb_iatt_fixup(this, entry->inode, &entry->d_stat);
+	}
+
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+static int32_t
+qb_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	    off_t off, dict_t *xdata)
+{
+	qb_conf_t *conf = this->private;
+
+	xdata = xdata ? dict_ref(xdata) : dict_new();
+	if (!xdata)
+		goto enomem;
+
+	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
+		goto enomem;
+
+	STACK_WIND(frame, qb_readdirp_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+	dict_unref(xdata);
+	return 0;
+
+enomem:
+	QB_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL);
+	if (xdata)
+		dict_unref(xdata);
+	return 0;
+}
+
+int
+qb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+	     dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, loc->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->truncate, loc, offset,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (loc->inode);
+	qb_local->fd = fd_anonymous (loc->inode);
+
+	qb_local->stub = fop_truncate_stub (frame, NULL, loc, offset, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_truncate);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (truncate, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	     dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->ftruncate, fd, offset,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_ftruncate_stub (frame, NULL, fd, offset, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_truncate);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	     int op_ret, int op_errno, struct iatt *iatt, dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, iatt);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (stat, frame, op_ret, op_errno, iatt, xdata);
+
+	return 0;
+}
+
+int
+qb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, loc->inode))
+		frame->local = inode_ref (loc->inode);
+
+	STACK_WIND (frame, qb_stat_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->stat, loc, xdata);
+	return 0;
+}
+
+
+int
+qb_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	      int op_ret, int op_errno, struct iatt *iatt, dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, iatt);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (fstat, frame, op_ret, op_errno, iatt, xdata);
+
+	return 0;
+}
+
+
+int
+qb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, fd->inode))
+		frame->local = inode_ref (fd->inode);
+
+	STACK_WIND (frame, qb_fstat_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+	return 0;
+}
+
+
+int
+qb_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+		dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, pre);
+		qb_iatt_fixup (this, inode, post);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (setattr, frame, op_ret, op_errno, pre, post, xdata);
+
+	return 0;
+}
+
+
+int
+qb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+	    int valid, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, loc->inode))
+		frame->local = inode_ref (loc->inode);
+
+	STACK_WIND (frame, qb_setattr_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->setattr, loc, buf, valid, xdata);
+	return 0;
+}
+
+
+int
+qb_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+		 dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, pre);
+		qb_iatt_fixup (this, inode, post);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, pre, post, xdata);
+
+	return 0;
+}
+
+
+int
+qb_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+	     int valid, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, fd->inode))
+		frame->local = inode_ref (fd->inode);
+
+	STACK_WIND (frame, qb_setattr_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fsetattr, fd, buf, valid, xdata);
+	return 0;
+}
+
+
+int
+qb_forget (xlator_t *this, inode_t *inode)
+{
+	return qb_inode_cleanup (this, inode, 0);
+}
+
+
+int
+qb_release (xlator_t *this, fd_t *fd)
+{
+	call_frame_t *frame = NULL;
+
+	frame = create_frame (this, this->ctx->pool);
+	if (!frame) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate frame. "
+			"Leaking QEMU BlockDriverState");
+		return -1;
+	}
+
+	if (qb_local_init (frame) != 0) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate local. "
+			"Leaking QEMU BlockDriverState");
+		STACK_DESTROY (frame->root);
+		return -1;
+	}
+
+	if (qb_coroutine (frame, qb_co_close) != 0) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate coroutine. "
+			"Leaking QEMU BlockDriverState");
+		qb_local_free (this, frame->local);
+		frame->local = NULL;
+		STACK_DESTROY (frame->root);
+	}
+
+	return 0;
+}
+
+int
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_qb_mt_end + 1);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init "
+                        "failed");
+        return ret;
+}
+
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+	return 0;
+}
+
+
+int
+init (xlator_t *this)
+{
+        qb_conf_t *conf    = NULL;
+        int32_t    ret     = -1;
+	static int bdrv_inited = 0;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "FATAL: qemu-block (%s) not configured with exactly "
+                        "one child", this->name);
+                goto out;
+        }
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_qb_mt_qb_conf_t);
+        if (!conf)
+                goto out;
+
+        /* configure 'option window-size <size>' */
+        GF_OPTION_INIT ("default-password", conf->default_password, str, out);
+
+	/* qemu coroutines use "co_mutex" for synchronizing among themselves.
+	   However "co_mutex" itself is not threadsafe if the coroutine framework
+	   is multithreaded (which usually is not). However synctasks are
+	   fundamentally multithreaded, so for now create a syncenv which has
+	   scaling limits set to max 1 thread so that the qemu coroutines can
+	   execute "safely".
+
+	   Future work: provide an implementation of "co_mutex" which is
+	   threadsafe and use the global multithreaded ctx->env syncenv.
+	*/
+	conf->env = syncenv_new (0, 1, 1);
+
+        this->private = conf;
+
+        ret = 0;
+
+	snprintf (conf->qb_xattr_key, QB_XATTR_KEY_MAX, QB_XATTR_KEY_FMT,
+		  this->name);
+
+	cur_mon = (void *) 1;
+
+	if (!bdrv_inited) {
+		bdrv_init ();
+		bdrv_inited = 1;
+	}
+
+out:
+        if (ret)
+                GF_FREE (conf);
+
+        return ret;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        qb_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        this->private = NULL;
+
+	if (conf->root_inode)
+		inode_unref(conf->root_inode);
+        GF_FREE (conf);
+
+	return;
+}
+
+
+struct xlator_fops fops = {
+	.lookup      = qb_lookup,
+	.fsetxattr   = qb_fsetxattr,
+	.setxattr    = qb_setxattr,
+	.open        = qb_open,
+        .writev      = qb_writev,
+        .readv       = qb_readv,
+        .fsync       = qb_fsync,
+        .truncate    = qb_truncate,
+        .ftruncate   = qb_ftruncate,
+        .stat        = qb_stat,
+        .fstat       = qb_fstat,
+	.setattr     = qb_setattr,
+	.fsetattr    = qb_fsetattr,
+        .flush       = qb_flush,
+/*
+	.getxattr    = qb_getxattr,
+	.fgetxattr   = qb_fgetxattr
+*/
+	.readdirp    = qb_readdirp,
+};
+
+
+struct xlator_cbks cbks = {
+        .forget   = qb_forget,
+	.release  = qb_release,
+};
+
+
+struct xlator_dumpops dumpops = {
+};
+
+
+struct volume_options options[] = {
+        { .key  = {"default-password"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "",
+          .description = "Default password for the AES encrypted block images."
+        },
+        { .key = {NULL} },
+};
diff --git a/xlators/features/qemu-block/src/qemu-block.h b/xlators/features/qemu-block/src/qemu-block.h
new file mode 100644
index 000000000..c95f2799a
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block.h
@@ -0,0 +1,109 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QEMU_BLOCK_H
+#define __QEMU_BLOCK_H
+
+#include "syncop.h"
+#include "call-stub.h"
+#include "block/block_int.h"
+#include "monitor/monitor.h"
+
+/* QB_XATTR_KEY_FMT is the on-disk xattr stored in the inode which
+   indicates that the file must be "interpreted" by the block format
+   logic. The value of the key is of the pattern:
+
+   "format:virtual_size"
+
+   e.g
+
+   "qcow2:20GB" or "qed:100GB"
+
+   The format and virtual size are colon separated. The format is
+   a case sensitive string which qemu recognizes. virtual_size is
+   specified as a size which glusterfs recognizes as size (i.e.,
+   value accepted by gf_string2bytesize())
+*/
+#define QB_XATTR_KEY_FMT "trusted.glusterfs.%s.format"
+
+#define QB_XATTR_KEY_MAX 64
+
+#define QB_XATTR_VAL_MAX 64
+
+
+typedef struct qb_inode {
+	char fmt[QB_XATTR_VAL_MAX]; /* this is only the format, not "format:size" */
+	size_t size; /* virtual size in bytes */
+	BlockDriverState *bs;
+	int refcnt;
+	uuid_t backing_gfid;
+	char *backing_fname;
+} qb_inode_t;
+
+
+typedef struct qb_conf {
+	Monitor *mon;
+	struct syncenv *env;
+	char qb_xattr_key[QB_XATTR_KEY_MAX];
+	char *default_password;
+	inode_t *root_inode;
+} qb_conf_t;
+
+
+typedef struct qb_local {
+	call_frame_t      *frame; /* backpointer */
+	call_stub_t       *stub;
+	inode_t           *inode;
+	fd_t              *fd;
+	char               fmt[QB_XATTR_VAL_MAX+1];
+	char               name[256];
+	synctask_fn_t      synctask_fn;
+	struct list_head   list;
+} qb_local_t;
+
+void qb_local_free (xlator_t *this, qb_local_t *local);
+int qb_coroutine (call_frame_t *frame, synctask_fn_t fn);
+inode_t *qb_inode_from_filename (const char *filename);
+int qb_inode_to_filename (inode_t *inode, char *filename, int size);
+int qb_format_extract (xlator_t *this, char *format, inode_t *inode);
+
+qb_inode_t *qb_inode_ctx_get (xlator_t *this, inode_t *inode);
+
+#define QB_STACK_UNWIND(typ, frame, args ...) do {	\
+	qb_local_t *__local = frame->local;		\
+	xlator_t *__this = frame->this;			\
+							\
+	frame->local = NULL;				\
+	STACK_UNWIND_STRICT (typ, frame, args);		\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#define QB_STUB_UNWIND(stub, op_ret, op_errno) do {	\
+	qb_local_t *__local = stub->frame->local;	\
+	xlator_t *__this = stub->frame->this;		\
+							\
+	stub->frame->local = NULL;			\
+	call_unwind_error (stub, op_ret, op_errno);	\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#define QB_STUB_RESUME(stub_errno) do {			\
+	qb_local_t *__local = stub->frame->local;	\
+	xlator_t *__this = stub->frame->this;		\
+							\
+	stub->frame->local = NULL;			\
+	call_resume (stub);				\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#endif /* !__QEMU_BLOCK_H */
diff --git a/xlators/features/quiesce/src/Makefile.am b/xlators/features/quiesce/src/Makefile.am
index e8ab4cb24..15e46629e 100644
--- a/xlators/features/quiesce/src/Makefile.am
+++ b/xlators/features/quiesce/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = quiesce.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-quiesce_la_LDFLAGS = -module -avoidversion
+quiesce_la_LDFLAGS = -module -avoid-version
 
 quiesce_la_SOURCES = quiesce.c
 quiesce_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = quiesce.h quiesce-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/features/quiesce/src/quiesce-mem-types.h b/xlators/features/quiesce/src/quiesce-mem-types.h
index 00f7aa4f1..6e582f424 100644
--- a/xlators/features/quiesce/src/quiesce-mem-types.h
+++ b/xlators/features/quiesce/src/quiesce-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __QUIESCE_MEM_TYPES_H__
 #define __QUIESCE_MEM_TYPES_H__
 
diff --git a/xlators/features/quiesce/src/quiesce.c b/xlators/features/quiesce/src/quiesce.c
index 1b150b215..24c7dc6ed 100644
--- a/xlators/features/quiesce/src/quiesce.c
+++ b/xlators/features/quiesce/src/quiesce.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -41,16 +31,13 @@ gf_quiesce_local_wipe (xlator_t *this, quiesce_local_t *local)
                 loc_wipe (&local->loc);
         if (local->fd)
                 fd_unref (local->fd);
-        if (local->name)
-                GF_FREE (local->name);
-        if (local->volname)
-                GF_FREE (local->volname);
+        GF_FREE (local->name);
+        GF_FREE (local->volname);
         if (local->dict)
                 dict_unref (local->dict);
         if (local->iobref)
                 iobref_unref (local->iobref);
-        if (local->vector)
-                GF_FREE (local->vector);
+        GF_FREE (local->vector);
 
         mem_put (local);
 }
@@ -124,7 +111,7 @@ void
 gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
 {
         quiesce_priv_t *priv    = NULL;
-        struct timeval  timeout = {0,};
+        struct timespec timeout = {0,};
 
         priv = this->private;
         if (!priv) {
@@ -142,7 +129,7 @@ gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
 
         if (!priv->timer) {
                 timeout.tv_sec = 20;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 priv->timer = gf_timer_call_after (this->ctx,
                                                    timeout,
@@ -191,7 +178,8 @@ out:
 
 int32_t
 quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -201,10 +189,10 @@ quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_stat_stub (frame, default_stat_resume,
-                                      &local->loc);
+                                      &local->loc, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -212,7 +200,7 @@ quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -221,7 +209,7 @@ out:
 
 int32_t
 quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -231,9 +219,9 @@ quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_access_stub (frame, default_access_resume,
-                                        &local->loc, local->flag);
+                                        &local->loc, local->flag, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (access, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -241,7 +229,7 @@ quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -251,7 +239,7 @@ out:
 int32_t
 quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf)
+                      struct iatt *buf, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -261,10 +249,10 @@ quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_readlink_stub (frame, default_readlink_resume,
-                                          &local->loc, local->size);
+                                          &local->loc, local->size, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -272,7 +260,7 @@ quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -281,7 +269,7 @@ out:
 
 int32_t
 quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd)
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -292,10 +280,10 @@ quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_open_stub (frame, default_open_resume,
                                       &local->loc, local->flag, local->fd,
-                                      local->wbflags);
+                                      xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (open, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -303,7 +291,7 @@ quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -313,7 +301,7 @@ out:
 int32_t
 quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -324,10 +312,10 @@ quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_readv_stub (frame, default_readv_resume,
                                        local->fd, local->size, local->offset,
-                                       local->io_flag);
+                                       local->io_flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
-                                             NULL, 0, NULL, NULL);
+                                             NULL, 0, NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -336,7 +324,7 @@ quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+                             stbuf, iobref, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -345,7 +333,7 @@ out:
 
 int32_t
 quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -355,9 +343,9 @@ quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_flush_stub (frame, default_flush_resume,
-                                       local->fd);
+                                       local->fd, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -365,7 +353,7 @@ quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -377,7 +365,7 @@ out:
 int32_t
 quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -387,10 +375,10 @@ quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fsync_stub (frame, default_fsync_resume,
-                                       local->fd, local->flag);
+                                       local->fd, local->flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -398,7 +386,7 @@ quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -407,7 +395,7 @@ out:
 
 int32_t
 quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -417,10 +405,10 @@ quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fstat_stub (frame, default_fstat_resume,
-                                       local->fd);
+                                       local->fd, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -428,7 +416,7 @@ quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -437,7 +425,7 @@ out:
 
 int32_t
 quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd)
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -447,10 +435,10 @@ quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_opendir_stub (frame, default_opendir_resume,
-                                         &local->loc, local->fd);
+                                         &local->loc, local->fd, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -458,7 +446,7 @@ quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -467,7 +455,7 @@ out:
 
 int32_t
 quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -477,9 +465,9 @@ quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume,
-                                          local->fd, local->flag);
+                                          local->fd, local->flag, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -487,7 +475,7 @@ quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -496,7 +484,7 @@ out:
 
 int32_t
 quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -506,10 +494,10 @@ quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_statfs_stub (frame, default_statfs_resume,
-                                        &local->loc);
+                                        &local->loc, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -517,7 +505,7 @@ quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -526,7 +514,7 @@ out:
 
 int32_t
 quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -536,10 +524,10 @@ quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume,
-                                           local->fd, local->name);
+                                           local->fd, local->name, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -547,7 +535,7 @@ quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -557,7 +545,7 @@ out:
 
 int32_t
 quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -567,10 +555,10 @@ quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_getxattr_stub (frame, default_getxattr_resume,
-                                          &local->loc, local->name);
+                                          &local->loc, local->name, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -578,7 +566,7 @@ quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -589,7 +577,7 @@ out:
 int32_t
 quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum)
+                       uint8_t *strong_checksum, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -599,10 +587,10 @@ quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
-                                           local->fd, local->offset, local->flag);
+                                           local->fd, local->offset, local->flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM,
-                                             0, NULL);
+                                             0, NULL, NULL);
                         goto out;
                 }
 
@@ -611,7 +599,7 @@ quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum);
+                             strong_checksum, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -621,7 +609,7 @@ out:
 
 int32_t
 quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -631,10 +619,10 @@ quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_readdir_stub (frame, default_readdir_resume,
-                                         local->fd, local->size, local->offset);
+                                         local->fd, local->size, local->offset, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -642,7 +630,7 @@ quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -652,7 +640,7 @@ out:
 
 int32_t
 quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         call_stub_t    *stub = NULL;
         quiesce_local_t *local = NULL;
@@ -666,7 +654,7 @@ quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                           local->dict);
                 if (!stub) {
                         STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -674,7 +662,7 @@ quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -687,7 +675,7 @@ out:
 int32_t
 quiesce_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf)
+                    struct iatt *postbuf, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -702,10 +690,10 @@ quiesce_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 stub = fop_writev_stub (frame, default_writev_resume,
                                         local->fd, local->vector, local->flag,
                                         local->offset, local->io_flags,
-                                        local->iobref);
+                                        local->iobref, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -713,7 +701,7 @@ quiesce_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -722,7 +710,7 @@ out:
 
 int32_t
 quiesce_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
+                     int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -736,10 +724,10 @@ quiesce_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_xattrop_stub (frame, default_xattrop_resume,
                                          &local->loc, local->xattrop_flags,
-                                         local->dict);
+                                         local->dict, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -747,7 +735,7 @@ quiesce_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -756,7 +744,7 @@ out:
 
 int32_t
 quiesce_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -770,10 +758,10 @@ quiesce_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fxattrop_stub (frame, default_fxattrop_resume,
                                           local->fd, local->xattrop_flags,
-                                          local->dict);
+                                          local->dict, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -781,7 +769,7 @@ quiesce_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -790,7 +778,7 @@ out:
 
 int32_t
 quiesce_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -803,10 +791,10 @@ quiesce_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_lk_stub (frame, default_lk_resume,
-                                    local->fd, local->flag, &local->flock);
+                                    local->fd, local->flag, &local->flock, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM,
-                                             NULL);
+                                             NULL, NULL);
                         goto out;
                 }
 
@@ -814,7 +802,7 @@ quiesce_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -823,7 +811,7 @@ out:
 
 int32_t
 quiesce_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -837,9 +825,9 @@ quiesce_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_inodelk_stub (frame, default_inodelk_resume,
                                          local->volname, &local->loc,
-                                         local->flag, &local->flock);
+                                         local->flag, &local->flock, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -847,7 +835,7 @@ quiesce_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -857,7 +845,7 @@ out:
 
 int32_t
 quiesce_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -871,9 +859,9 @@ quiesce_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_finodelk_stub (frame, default_finodelk_resume,
                                          local->volname, local->fd,
-                                         local->flag, &local->flock);
+                                         local->flag, &local->flock, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -881,7 +869,7 @@ quiesce_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -890,7 +878,7 @@ out:
 
 int32_t
 quiesce_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -904,9 +892,9 @@ quiesce_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_entrylk_stub (frame, default_entrylk_resume,
                                          local->volname, &local->loc,
-                                         local->name, local->cmd, local->type);
+                                         local->name, local->cmd, local->type, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -914,7 +902,7 @@ quiesce_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -923,7 +911,7 @@ out:
 
 int32_t
 quiesce_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -937,9 +925,9 @@ quiesce_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fentrylk_stub (frame, default_fentrylk_resume,
                                           local->volname, local->fd,
-                                          local->name, local->cmd, local->type);
+                                          local->name, local->cmd, local->type, xdata);
                 if (!stub) {
-                        STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM);
+                        STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, NULL);
                         goto out;
                 }
 
@@ -947,7 +935,7 @@ quiesce_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -957,7 +945,7 @@ out:
 int32_t
 quiesce_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost)
+                     struct iatt *statpost, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -970,10 +958,10 @@ quiesce_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_setattr_stub (frame, default_setattr_resume,
-                                         &local->loc, &local->stbuf, local->flag);
+                                         &local->loc, &local->stbuf, local->flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -982,7 +970,7 @@ quiesce_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -992,7 +980,7 @@ out:
 int32_t
 quiesce_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                      struct iatt *statpost)
+                      struct iatt *statpost, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1006,10 +994,10 @@ quiesce_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && (op_errno == ENOTCONN)) {
                 /* Re-transmit (by putting in the queue) */
                 stub = fop_fsetattr_stub (frame, default_fsetattr_resume,
-                                          local->fd, &local->stbuf, local->flag);
+                                          local->fd, &local->stbuf, local->flag, xdata);
                 if (!stub) {
                         STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM,
-                                             NULL, NULL);
+                                             NULL, NULL, NULL);
                         goto out;
                 }
 
@@ -1018,7 +1006,7 @@ quiesce_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
-                             statpost);
+                             statpost, xdata);
 out:
         gf_quiesce_local_wipe (this, local);
 
@@ -1036,7 +1024,7 @@ int32_t
 quiesce_removexattr (call_frame_t *frame,
 		     xlator_t *this,
 		     loc_t *loc,
-		     const char *name)
+		     const char *name, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1049,14 +1037,14 @@ quiesce_removexattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->removexattr,
                             loc,
-                            name);
-	        return 0;
+                            name, xdata);
+                return 0;
         }
 
         stub = fop_removexattr_stub (frame, default_removexattr_resume,
-                                     loc, name);
+                                     loc, name, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1069,7 +1057,7 @@ int32_t
 quiesce_truncate (call_frame_t *frame,
 		  xlator_t *this,
 		  loc_t *loc,
-		  off_t offset)
+		  off_t offset, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1082,13 +1070,13 @@ quiesce_truncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->truncate,
                             loc,
-                            offset);
-	        return 0;
+                            offset, xdata);
+                return 0;
         }
 
-        stub = fop_truncate_stub (frame, default_truncate_resume, loc, offset);
+        stub = fop_truncate_stub (frame, default_truncate_resume, loc, offset, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1102,7 +1090,7 @@ quiesce_fsetxattr (call_frame_t *frame,
                    xlator_t *this,
                    fd_t *fd,
                    dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1116,14 +1104,14 @@ quiesce_fsetxattr (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->fsetxattr,
                             fd,
                             dict,
-                            flags);
-	        return 0;
+                            flags, xdata);
+                return 0;
         }
 
         stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume,
-                                   fd, dict, flags);
+                                   fd, dict, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1137,7 +1125,7 @@ quiesce_setxattr (call_frame_t *frame,
 		  xlator_t *this,
 		  loc_t *loc,
 		  dict_t *dict,
-		  int32_t flags)
+		  int32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1151,14 +1139,14 @@ quiesce_setxattr (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->setxattr,
                             loc,
                             dict,
-                            flags);
-	        return 0;
+                            flags, xdata);
+                return 0;
         }
 
         stub = fop_setxattr_stub (frame, default_setxattr_resume,
-                                  loc, dict, flags);
+                                  loc, dict, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1170,7 +1158,7 @@ quiesce_setxattr (call_frame_t *frame,
 int32_t
 quiesce_create (call_frame_t *frame, xlator_t *this,
 		loc_t *loc, int32_t flags, mode_t mode,
-                fd_t *fd, dict_t *params)
+                mode_t umask, fd_t *fd, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1183,15 +1171,15 @@ quiesce_create (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_create_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->create,
-                            loc, (flags & ~O_APPEND), mode, fd, params);
-	        return 0;
+                            loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
+                return 0;
         }
 
         stub = fop_create_stub (frame, default_create_resume,
-                                loc, (flags & ~O_APPEND), mode, fd, params);
+                                loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (create, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1204,7 +1192,7 @@ int32_t
 quiesce_link (call_frame_t *frame,
 	      xlator_t *this,
 	      loc_t *oldloc,
-	      loc_t *newloc)
+	      loc_t *newloc, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1216,14 +1204,14 @@ quiesce_link (call_frame_t *frame,
                             default_link_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc);
-	        return 0;
+                            oldloc, newloc, xdata);
+                return 0;
         }
 
-        stub = fop_link_stub (frame, default_link_resume, oldloc, newloc);
+        stub = fop_link_stub (frame, default_link_resume, oldloc, newloc, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (link, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1236,7 +1224,7 @@ int32_t
 quiesce_rename (call_frame_t *frame,
 		xlator_t *this,
 		loc_t *oldloc,
-		loc_t *newloc)
+		loc_t *newloc, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1248,14 +1236,14 @@ quiesce_rename (call_frame_t *frame,
                             default_rename_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc);
-	        return 0;
+                            oldloc, newloc, xdata);
+                return 0;
         }
 
-        stub = fop_rename_stub (frame, default_rename_resume, oldloc, newloc);
+        stub = fop_rename_stub (frame, default_rename_resume, oldloc, newloc, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1267,7 +1255,7 @@ quiesce_rename (call_frame_t *frame,
 
 int
 quiesce_symlink (call_frame_t *frame, xlator_t *this,
-		 const char *linkpath, loc_t *loc, dict_t *params)
+		 const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1278,15 +1266,15 @@ quiesce_symlink (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_symlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, params);
-	        return 0;
+                            linkpath, loc, umask, xdata);
+                return 0;
         }
 
         stub = fop_symlink_stub (frame, default_symlink_resume,
-                                 linkpath, loc, params);
+                                 linkpath, loc, umask, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1297,7 +1285,7 @@ quiesce_symlink (call_frame_t *frame, xlator_t *this,
 
 
 int
-quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1308,13 +1296,13 @@ quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
                 STACK_WIND (frame, default_rmdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags);
-	        return 0;
+                            loc, flags, xdata);
+                return 0;
         }
 
-        stub = fop_rmdir_stub (frame, default_rmdir_resume, loc, flags);
+        stub = fop_rmdir_stub (frame, default_rmdir_resume, loc, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1326,7 +1314,7 @@ quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 int32_t
 quiesce_unlink (call_frame_t *frame,
 		xlator_t *this,
-		loc_t *loc)
+		loc_t *loc, int xflag, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1338,13 +1326,13 @@ quiesce_unlink (call_frame_t *frame,
                             default_unlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->unlink,
-                            loc);
-	        return 0;
+                            loc, xflag, xdata);
+                return 0;
         }
 
-        stub = fop_unlink_stub (frame, default_unlink_resume, loc);
+        stub = fop_unlink_stub (frame, default_unlink_resume, loc, xflag, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1355,7 +1343,7 @@ quiesce_unlink (call_frame_t *frame,
 
 int
 quiesce_mkdir (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, mode_t mode, dict_t *params)
+	       loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1366,15 +1354,15 @@ quiesce_mkdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mkdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, params);
-	        return 0;
+                            loc, mode, umask, xdata);
+                return 0;
         }
 
         stub = fop_mkdir_stub (frame, default_mkdir_resume,
-                               loc, mode, params);
+                               loc, mode, umask, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1386,7 +1374,7 @@ quiesce_mkdir (call_frame_t *frame, xlator_t *this,
 
 int
 quiesce_mknod (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, mode_t mode, dev_t rdev, dict_t *parms)
+	       loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1397,15 +1385,15 @@ quiesce_mknod (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mknod_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, parms);
-	        return 0;
+                            loc, mode, rdev, umask, xdata);
+                return 0;
         }
 
         stub = fop_mknod_stub (frame, default_mknod_resume,
-                               loc, mode, rdev, parms);
+                               loc, mode, rdev, umask, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1418,7 +1406,7 @@ int32_t
 quiesce_ftruncate (call_frame_t *frame,
 		   xlator_t *this,
 		   fd_t *fd,
-		   off_t offset)
+		   off_t offset, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1431,13 +1419,13 @@ quiesce_ftruncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->ftruncate,
                             fd,
-                            offset);
-	        return 0;
+                            offset, xdata);
+                return 0;
         }
 
-        stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset);
+        stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1452,7 +1440,7 @@ int32_t
 quiesce_readlink (call_frame_t *frame,
 		  xlator_t *this,
 		  loc_t *loc,
-		  size_t size)
+		  size_t size, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1471,13 +1459,13 @@ quiesce_readlink (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readlink,
                             loc,
-                            size);
-	        return 0;
+                            size, xdata);
+                return 0;
         }
 
-        stub = fop_readlink_stub (frame, default_readlink_resume, loc, size);
+        stub = fop_readlink_stub (frame, default_readlink_resume, loc, size, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1491,7 +1479,7 @@ int32_t
 quiesce_access (call_frame_t *frame,
 		xlator_t *this,
 		loc_t *loc,
-		int32_t mask)
+		int32_t mask, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1510,13 +1498,13 @@ quiesce_access (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->access,
                             loc,
-                            mask);
-	        return 0;
+                            mask, xdata);
+                return 0;
         }
 
-        stub = fop_access_stub (frame, default_access_resume, loc, mask);
+        stub = fop_access_stub (frame, default_access_resume, loc, mask, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (access, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1529,7 +1517,7 @@ int32_t
 quiesce_fgetxattr (call_frame_t *frame,
                    xlator_t *this,
                    fd_t *fd,
-                   const char *name)
+                   const char *name, dict_t *xdata)
 {
         quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1550,13 +1538,13 @@ quiesce_fgetxattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fgetxattr,
                             fd,
-                            name);
+                            name, xdata);
                 return 0;
         }
 
-        stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name);
+        stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1568,7 +1556,7 @@ quiesce_fgetxattr (call_frame_t *frame,
 int32_t
 quiesce_statfs (call_frame_t *frame,
 		xlator_t *this,
-		loc_t *loc)
+		loc_t *loc, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1585,13 +1573,13 @@ quiesce_statfs (call_frame_t *frame,
                             quiesce_statfs_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->statfs,
-                            loc);
-	        return 0;
+                            loc, xdata);
+                return 0;
         }
 
-        stub = fop_statfs_stub (frame, default_statfs_resume, loc);
+        stub = fop_statfs_stub (frame, default_statfs_resume, loc, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1604,7 +1592,7 @@ int32_t
 quiesce_fsyncdir (call_frame_t *frame,
 		  xlator_t *this,
 		  fd_t *fd,
-		  int32_t flags)
+		  int32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1623,13 +1611,13 @@ quiesce_fsyncdir (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsyncdir,
                             fd,
-                            flags);
-	        return 0;
+                            flags, xdata);
+                return 0;
         }
 
-        stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume, fd, flags);
+        stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume, fd, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1641,7 +1629,7 @@ quiesce_fsyncdir (call_frame_t *frame,
 int32_t
 quiesce_opendir (call_frame_t *frame,
 		 xlator_t *this,
-		 loc_t *loc, fd_t *fd)
+		 loc_t *loc, fd_t *fd, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1659,13 +1647,13 @@ quiesce_opendir (call_frame_t *frame,
                             quiesce_opendir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->opendir,
-                            loc, fd);
-	        return 0;
+                            loc, fd, xdata);
+                return 0;
         }
 
-        stub = fop_opendir_stub (frame, default_opendir_resume, loc, fd);
+        stub = fop_opendir_stub (frame, default_opendir_resume, loc, fd, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1677,7 +1665,7 @@ quiesce_opendir (call_frame_t *frame,
 int32_t
 quiesce_fstat (call_frame_t *frame,
 	       xlator_t *this,
-	       fd_t *fd)
+	       fd_t *fd, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1694,13 +1682,13 @@ quiesce_fstat (call_frame_t *frame,
                             quiesce_fstat_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fstat,
-                            fd);
-	        return 0;
+                            fd, xdata);
+                return 0;
         }
 
-        stub = fop_fstat_stub (frame, default_fstat_resume, fd);
+        stub = fop_fstat_stub (frame, default_fstat_resume, fd, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1713,7 +1701,7 @@ int32_t
 quiesce_fsync (call_frame_t *frame,
 	       xlator_t *this,
 	       fd_t *fd,
-	       int32_t flags)
+	       int32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1732,13 +1720,13 @@ quiesce_fsync (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsync,
                             fd,
-                            flags);
-	        return 0;
+                            flags, xdata);
+                return 0;
         }
 
-        stub = fop_fsync_stub (frame, default_fsync_resume, fd, flags);
+        stub = fop_fsync_stub (frame, default_fsync_resume, fd, flags, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1750,7 +1738,7 @@ quiesce_fsync (call_frame_t *frame,
 int32_t
 quiesce_flush (call_frame_t *frame,
 	       xlator_t *this,
-	       fd_t *fd)
+	       fd_t *fd, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1767,13 +1755,13 @@ quiesce_flush (call_frame_t *frame,
                             quiesce_flush_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->flush,
-                            fd);
-	        return 0;
+                            fd, xdata);
+                return 0;
         }
 
-        stub = fop_flush_stub (frame, default_flush_resume, fd);
+        stub = fop_flush_stub (frame, default_flush_resume, fd, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -1789,7 +1777,7 @@ quiesce_writev (call_frame_t *frame,
 		struct iovec *vector,
 		int32_t count,
 		off_t off, uint32_t flags,
-                struct iobref *iobref)
+                struct iobref *iobref, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1805,14 +1793,14 @@ quiesce_writev (call_frame_t *frame,
                             vector,
                             count,
                             off, flags,
-                            iobref);
-	        return 0;
+                            iobref, xdata);
+                return 0;
         }
 
         stub = fop_writev_stub (frame, default_writev_resume,
-                                fd, vector, count, off, flags, iobref);
+                                fd, vector, count, off, flags, iobref, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1826,7 +1814,7 @@ quiesce_readv (call_frame_t *frame,
 	       xlator_t *this,
 	       fd_t *fd,
 	       size_t size,
-	       off_t offset, uint32_t flags)
+	       off_t offset, uint32_t flags, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1848,15 +1836,15 @@ quiesce_readv (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->readv,
                             fd,
                             size,
-                            offset, flags);
-	        return 0;
+                            offset, flags, xdata);
+                return 0;
         }
 
         stub = fop_readv_stub (frame, default_readv_resume, fd, size, offset,
-                               flags);
+                               flags, xdata);
         if (!stub) {
                 STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
-                                     NULL, 0, NULL, NULL);
+                                     NULL, 0, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1871,7 +1859,7 @@ quiesce_open (call_frame_t *frame,
 	      xlator_t *this,
 	      loc_t *loc,
 	      int32_t flags, fd_t *fd,
-              int32_t wbflags)
+              dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1887,21 +1875,20 @@ quiesce_open (call_frame_t *frame,
                 /* Don't send O_APPEND below, as write() re-transmittions can
                    fail with O_APPEND */
                 local->flag = (flags & ~O_APPEND);
-                local->wbflags = wbflags;
                 frame->local = local;
 
                 STACK_WIND (frame,
                             quiesce_open_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->open,
-                            loc, (flags & ~O_APPEND), fd, wbflags);
-	        return 0;
+                            loc, (flags & ~O_APPEND), fd, xdata);
+                return 0;
         }
 
         stub = fop_open_stub (frame, default_open_resume, loc,
-                              (flags & ~O_APPEND), fd, wbflags);
+                              (flags & ~O_APPEND), fd, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1914,7 +1901,7 @@ int32_t
 quiesce_getxattr (call_frame_t *frame,
 		  xlator_t *this,
 		  loc_t *loc,
-		  const char *name)
+		  const char *name, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1935,13 +1922,13 @@ quiesce_getxattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->getxattr,
                             loc,
-                            name);
-	        return 0;
+                            name, xdata);
+                return 0;
         }
 
-        stub = fop_getxattr_stub (frame, default_getxattr_resume, loc, name);
+        stub = fop_getxattr_stub (frame, default_getxattr_resume, loc, name, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1956,7 +1943,7 @@ quiesce_xattrop (call_frame_t *frame,
 		 xlator_t *this,
 		 loc_t *loc,
 		 gf_xattrop_flags_t flags,
-		 dict_t *dict)
+		 dict_t *dict, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -1970,14 +1957,14 @@ quiesce_xattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->xattrop,
                             loc,
                             flags,
-                            dict);
-	        return 0;
+                            dict, xdata);
+                return 0;
         }
 
         stub = fop_xattrop_stub (frame, default_xattrop_resume,
-                                 loc, flags, dict);
+                                 loc, flags, dict, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -1991,7 +1978,7 @@ quiesce_fxattrop (call_frame_t *frame,
 		  xlator_t *this,
 		  fd_t *fd,
 		  gf_xattrop_flags_t flags,
-		  dict_t *dict)
+		  dict_t *dict, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2005,14 +1992,14 @@ quiesce_fxattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->fxattrop,
                             fd,
                             flags,
-                            dict);
-	        return 0;
+                            dict, xdata);
+                return 0;
         }
 
         stub = fop_fxattrop_stub (frame, default_fxattrop_resume,
-                                  fd, flags, dict);
+                                  fd, flags, dict, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2026,7 +2013,7 @@ quiesce_lk (call_frame_t *frame,
 	    xlator_t *this,
 	    fd_t *fd,
 	    int32_t cmd,
-	    struct gf_flock *lock)
+	    struct gf_flock *lock, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2040,13 +2027,13 @@ quiesce_lk (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->lk,
                             fd,
                             cmd,
-                            lock);
-	        return 0;
+                            lock, xdata);
+                return 0;
         }
 
-        stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, lock);
+        stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, lock, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2059,7 +2046,7 @@ quiesce_lk (call_frame_t *frame,
 int32_t
 quiesce_inodelk (call_frame_t *frame, xlator_t *this,
 		 const char *volume, loc_t *loc, int32_t cmd,
-                 struct gf_flock *lock)
+                 struct gf_flock *lock, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2071,14 +2058,14 @@ quiesce_inodelk (call_frame_t *frame, xlator_t *this,
                             default_inodelk_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->inodelk,
-                            volume, loc, cmd, lock);
-	        return 0;
+                            volume, loc, cmd, lock, xdata);
+                return 0;
         }
 
         stub = fop_inodelk_stub (frame, default_inodelk_resume,
-                                 volume, loc, cmd, lock);
+                                 volume, loc, cmd, lock, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -2089,7 +2076,7 @@ quiesce_inodelk (call_frame_t *frame, xlator_t *this,
 
 int32_t
 quiesce_finodelk (call_frame_t *frame, xlator_t *this,
-		  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+		  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2101,14 +2088,14 @@ quiesce_finodelk (call_frame_t *frame, xlator_t *this,
                             default_finodelk_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->finodelk,
-                            volume, fd, cmd, lock);
-	        return 0;
+                            volume, fd, cmd, lock, xdata);
+                return 0;
         }
 
         stub = fop_finodelk_stub (frame, default_finodelk_resume,
-                                  volume, fd, cmd, lock);
+                                  volume, fd, cmd, lock, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -2120,7 +2107,7 @@ quiesce_finodelk (call_frame_t *frame, xlator_t *this,
 int32_t
 quiesce_entrylk (call_frame_t *frame, xlator_t *this,
 		 const char *volume, loc_t *loc, const char *basename,
-		 entrylk_cmd cmd, entrylk_type type)
+		 entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2131,14 +2118,14 @@ quiesce_entrylk (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_entrylk_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->entrylk,
-                            volume, loc, basename, cmd, type);
-	        return 0;
+                            volume, loc, basename, cmd, type, xdata);
+                return 0;
         }
 
         stub = fop_entrylk_stub (frame, default_entrylk_resume,
-                                 volume, loc, basename, cmd, type);
+                                 volume, loc, basename, cmd, type, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -2150,7 +2137,7 @@ quiesce_entrylk (call_frame_t *frame, xlator_t *this,
 int32_t
 quiesce_fentrylk (call_frame_t *frame, xlator_t *this,
 		  const char *volume, fd_t *fd, const char *basename,
-		  entrylk_cmd cmd, entrylk_type type)
+		  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2161,14 +2148,14 @@ quiesce_fentrylk (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_fentrylk_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fentrylk,
-                            volume, fd, basename, cmd, type);
-	        return 0;
+                            volume, fd, basename, cmd, type, xdata);
+                return 0;
         }
 
         stub = fop_fentrylk_stub (frame, default_fentrylk_resume,
-                                  volume, fd, basename, cmd, type);
+                                  volume, fd, basename, cmd, type, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM);
+                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, NULL);
                 return 0;
         }
 
@@ -2181,7 +2168,7 @@ int32_t
 quiesce_rchecksum (call_frame_t *frame,
                    xlator_t *this,
                    fd_t *fd, off_t offset,
-                   int32_t len)
+                   int32_t len, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2200,14 +2187,14 @@ quiesce_rchecksum (call_frame_t *frame,
                             quiesce_rchecksum_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rchecksum,
-                            fd, offset, len);
-	        return 0;
+                            fd, offset, len, xdata);
+                return 0;
         }
 
         stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
-                                   fd, offset, len);
+                                   fd, offset, len, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM, 0, NULL);
+                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
                 return 0;
         }
 
@@ -2222,7 +2209,7 @@ quiesce_readdir (call_frame_t *frame,
 		 xlator_t *this,
 		 fd_t *fd,
 		 size_t size,
-		 off_t off)
+		 off_t off, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2241,13 +2228,13 @@ quiesce_readdir (call_frame_t *frame,
                             quiesce_readdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off);
-	        return 0;
+                            fd, size, off, xdata);
+                return 0;
         }
 
-        stub = fop_readdir_stub (frame, default_readdir_resume, fd, size, off);
+        stub = fop_readdir_stub (frame, default_readdir_resume, fd, size, off, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2283,13 +2270,13 @@ quiesce_readdirp (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdirp,
                             fd, size, off, dict);
-	        return 0;
+                return 0;
         }
 
         stub = fop_readdirp_stub (frame, default_readdirp_resume, fd, size,
                                   off, dict);
         if (!stub) {
-                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2303,7 +2290,7 @@ quiesce_setattr (call_frame_t *frame,
                  xlator_t *this,
                  loc_t *loc,
                  struct iatt *stbuf,
-                 int32_t valid)
+                 int32_t valid, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2315,14 +2302,14 @@ quiesce_setattr (call_frame_t *frame,
                             default_setattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid);
-	        return 0;
+                            loc, stbuf, valid, xdata);
+                return 0;
         }
 
         stub = fop_setattr_stub (frame, default_setattr_resume,
-                                   loc, stbuf, valid);
+                                   loc, stbuf, valid, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -2335,7 +2322,7 @@ quiesce_setattr (call_frame_t *frame,
 int32_t
 quiesce_stat (call_frame_t *frame,
 	      xlator_t *this,
-	      loc_t *loc)
+	      loc_t *loc, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2352,13 +2339,13 @@ quiesce_stat (call_frame_t *frame,
                             quiesce_stat_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->stat,
-                            loc);
-	        return 0;
+                            loc, xdata);
+                return 0;
         }
 
-        stub = fop_stat_stub (frame, default_stat_resume, loc);
+        stub = fop_stat_stub (frame, default_stat_resume, loc, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
@@ -2389,9 +2376,8 @@ quiesce_lookup (call_frame_t *frame,
                             quiesce_lookup_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->lookup,
-                            loc,
-                            xattr_req);
-	        return 0;
+                            loc, xattr_req);
+                return 0;
         }
 
         stub = fop_lookup_stub (frame, default_lookup_resume, loc, xattr_req);
@@ -2411,7 +2397,7 @@ quiesce_fsetattr (call_frame_t *frame,
                   xlator_t *this,
                   fd_t *fd,
                   struct iatt *stbuf,
-                  int32_t valid)
+                  int32_t valid, dict_t *xdata)
 {
 	quiesce_priv_t *priv = NULL;
         call_stub_t    *stub = NULL;
@@ -2423,14 +2409,14 @@ quiesce_fsetattr (call_frame_t *frame,
                             default_fsetattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid);
-	        return 0;
+                            fd, stbuf, valid, xdata);
+                return 0;
         }
 
         stub = fop_fsetattr_stub (frame, default_fsetattr_resume,
-                                  fd, stbuf, valid);
+                                  fd, stbuf, valid, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -2506,7 +2492,7 @@ notify (xlator_t *this, int event, void *data, ...)
 {
         int             ret     = 0;
         quiesce_priv_t *priv    = NULL;
-        struct timeval  timeout = {0,};
+        struct timespec timeout = {0,};
 
         priv = this->private;
         if (!priv)
@@ -2539,7 +2525,7 @@ notify (xlator_t *this, int event, void *data, ...)
                 if (priv->timer)
                         break;
                 timeout.tv_sec = 20;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 priv->timer = gf_timer_call_after (this->ctx,
                                                    timeout,
@@ -2613,12 +2599,10 @@ struct xlator_fops fops = {
 
 };
 
-struct xlator_dumpops dumpops = {
-};
+struct xlator_dumpops dumpops;
 
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 
 struct volume_options options[] = {
diff --git a/xlators/features/quiesce/src/quiesce.h b/xlators/features/quiesce/src/quiesce.h
index 08f87e2ae..878ed77e9 100644
--- a/xlators/features/quiesce/src/quiesce.h
+++ b/xlators/features/quiesce/src/quiesce.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __QUIESCE_H__
 #define __QUIESCE_H__
 
diff --git a/xlators/features/quota/src/Makefile.am b/xlators/features/quota/src/Makefile.am
index 4baa5f06e..7165adc59 100644
--- a/xlators/features/quota/src/Makefile.am
+++ b/xlators/features/quota/src/Makefile.am
@@ -1,16 +1,22 @@
-xlator_LTLIBRARIES = quota.la
+xlator_LTLIBRARIES = quota.la quotad.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-quota_la_LDFLAGS = -module -avoidversion
+quota_la_LDFLAGS = -module -avoid-version
+quotad_la_LDFLAGS = -module -avoid-version
 
-quota_la_SOURCES = quota.c
+quota_la_SOURCES = quota.c quota-enforcer-client.c
 quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = quota-mem-types.h quota.h
+quotad_la_SOURCES = quotad.c quotad-helpers.c quotad-aggregator.c
+quotad_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
-	-I$(top_srcdir)/xlators/cluster/dht/src
+noinst_HEADERS = quota-mem-types.h quota.h quotad-aggregator.h quotad-helpers.h
 
-CLEANFILES = 
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/cluster/dht/src -I$(top_srcdir)/rpc/xdr/src/ \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
 
diff --git a/xlators/features/quota/src/quota-enforcer-client.c b/xlators/features/quota/src/quota-enforcer-client.c
new file mode 100644
index 000000000..7d8ab937d
--- /dev/null
+++ b/xlators/features/quota/src/quota-enforcer-client.c
@@ -0,0 +1,403 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/resource.h>
+#include <sys/file.h>
+#include <netdb.h>
+#include <signal.h>
+#include <libgen.h>
+
+#include <sys/utsname.h>
+
+#include <stdint.h>
+#include <pthread.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <time.h>
+#include <semaphore.h>
+#include <errno.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#ifdef HAVE_MALLOC_H
+#include <malloc.h>
+#endif
+
+#ifdef HAVE_MALLOC_STATS
+#ifdef DEBUG
+#include <mcheck.h>
+#endif
+#endif
+
+#include "quota.h"
+
+extern struct rpc_clnt_program quota_enforcer_clnt;
+
+int32_t
+quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent);
+
+int
+quota_enforcer_submit_request (void *req, call_frame_t *frame,
+                               rpc_clnt_prog_t *prog,
+                               int procnum, struct iobref *iobref,
+                               xlator_t *this, fop_cbk_fn_t cbkfn,
+                               xdrproc_t xdrproc)
+{
+        int           ret        = -1;
+        int           count      = 0;
+        struct iovec  iov        = {0, };
+        struct iobuf *iobuf      = NULL;
+        char          new_iobref = 0;
+        ssize_t       xdr_size   = 0;
+        quota_priv_t *priv       = NULL;
+
+        GF_ASSERT (this);
+
+        priv = this->private;
+
+        if (req) {
+                xdr_size = xdr_sizeof (xdrproc, req);
+                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
+                if (!iobuf) {
+                        goto out;
+                }
+
+                if (!iobref) {
+                        iobref = iobref_new ();
+                        if (!iobref) {
+                                goto out;
+                        }
+
+                        new_iobref = 1;
+                }
+
+                iobref_add (iobref, iobuf);
+
+                iov.iov_base = iobuf->ptr;
+                iov.iov_len  = iobuf_size (iobuf);
+
+                /* Create the xdr payload */
+                ret = xdr_serialize_generic (iov, req, xdrproc);
+                if (ret == -1) {
+                        goto out;
+                }
+                iov.iov_len = ret;
+                count = 1;
+        }
+
+        /* Send the msg */
+        ret = rpc_clnt_submit (priv->rpc_clnt, prog, procnum, cbkfn,
+                               &iov, count,
+                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+        ret = 0;
+
+out:
+        if (new_iobref)
+                iobref_unref (iobref);
+        if (iobuf)
+                iobuf_unref (iobuf);
+
+        return ret;
+}
+
+int
+quota_enforcer_lookup_cbk (struct rpc_req *req, struct iovec *iov,
+                           int count, void *myframe)
+{
+        quota_local_t    *local      = NULL;
+        call_frame_t     *frame      = NULL;
+        int               ret        = 0;
+        gfs3_lookup_rsp   rsp        = {0,};
+        struct iatt       stbuf      = {0,};
+        struct iatt       postparent = {0,};
+        int               op_errno   = EINVAL;
+        dict_t           *xdata      = NULL;
+        inode_t          *inode      = NULL;
+        xlator_t         *this       = NULL;
+
+        this = THIS;
+
+        frame = myframe;
+        local = frame->local;
+        inode = local->validate_loc.inode;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lookup_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        op_errno = gf_error_to_errno (rsp.op_errno);
+        gf_stat_to_iatt (&rsp.postparent, &postparent);
+
+        if (rsp.op_ret == -1)
+                goto out;
+
+        rsp.op_ret = -1;
+        gf_stat_to_iatt (&rsp.stat, &stbuf);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), rsp.op_ret,
+                                      op_errno, out);
+
+        if ((!uuid_is_null (inode->gfid))
+            && (uuid_compare (stbuf.ia_gfid, inode->gfid) != 0)) {
+                gf_log (frame->this->name, GF_LOG_DEBUG,
+                        "gfid changed for %s", local->validate_loc.path);
+                rsp.op_ret = -1;
+                op_errno = ESTALE;
+                goto out;
+        }
+
+        rsp.op_ret = 0;
+
+out:
+        rsp.op_errno = op_errno;
+        if (rsp.op_ret == -1) {
+                /* any error other than ENOENT */
+                if (rsp.op_errno != ENOENT)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "remote operation failed: %s. Path: %s (%s)",
+                                strerror (rsp.op_errno),
+                                local->validate_loc.path,
+                                loc_gfid_utoa (&local->validate_loc));
+                else
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "not found on remote node");
+
+        }
+
+        local->validate_cbk (frame, NULL, this, rsp.op_ret, rsp.op_errno, inode,
+                             &stbuf, xdata, &postparent);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata, fop_lookup_cbk_t validate_cbk)
+{
+        quota_local_t          *local      = NULL;
+        gfs3_lookup_req         req        = {{0,},};
+        int                     ret        = 0;
+        int                     op_errno   = ESTALE;
+        quota_priv_t           *priv       = NULL;
+
+        if (!frame || !this || !loc)
+                goto unwind;
+
+        local = frame->local;
+        local->validate_cbk = validate_cbk;
+
+        priv = this->private;
+
+        if (!(loc && loc->inode))
+                goto unwind;
+
+        if (!uuid_is_null (loc->inode->gfid))
+                memcpy (req.gfid, loc->inode->gfid, 16);
+        else
+                memcpy (req.gfid, loc->gfid, 16);
+
+        if (xdata) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, xdata,
+                                            (&req.xdata.xdata_val),
+                                            req.xdata.xdata_len,
+                                            op_errno, unwind);
+        }
+
+        if (loc->name)
+                req.bname = (char *)loc->name;
+        else
+                req.bname = "";
+
+        ret = quota_enforcer_submit_request (&req, frame,
+                                             priv->quota_enforcer,
+                                             GF_AGGREGATOR_LOOKUP,
+                                             NULL, this,
+                                             quota_enforcer_lookup_cbk,
+                                             (xdrproc_t)xdr_gfs3_lookup_req);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+        }
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+
+unwind:
+        validate_cbk (frame, NULL, this, -1, op_errno, NULL, NULL, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+quota_enforcer_notify (struct rpc_clnt *rpc, void *mydata,
+                       rpc_clnt_event_t event, void *data)
+{
+        xlator_t                *this = NULL;
+        int                     ret = 0;
+
+        this = mydata;
+
+        switch (event) {
+        case RPC_CLNT_CONNECT:
+        {
+                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
+                break;
+        }
+
+        case RPC_CLNT_DISCONNECT:
+        {
+                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
+                break;
+        }
+
+        default:
+                gf_log (this->name, GF_LOG_TRACE,
+                        "got some other RPC event %d", event);
+                ret = 0;
+                break;
+        }
+
+        return ret;
+}
+
+int
+quota_enforcer_blocking_connect (rpc_clnt_t *rpc)
+{
+        dict_t *options = NULL;
+        int     ret     = -1;
+
+        options = dict_new ();
+        if (options == NULL)
+                goto out;
+
+        ret = dict_set_str (options, "non-blocking-io", "no");
+        if (ret)
+                goto out;
+
+        rpc->conn.trans->reconfigure (rpc->conn.trans, options);
+
+        rpc_clnt_start (rpc);
+
+        ret = dict_set_str (options, "non-blocking-io", "yes");
+        if (ret)
+                goto out;
+
+        rpc->conn.trans->reconfigure (rpc->conn.trans, options);
+
+        ret = 0;
+out:
+        dict_unref (options);
+
+        return ret;
+}
+
+//Returns a started rpc_clnt. Creates a new rpc_clnt if quota_priv doesn't have
+//one already
+struct rpc_clnt *
+quota_enforcer_init (xlator_t *this, dict_t *options)
+{
+        struct rpc_clnt *rpc  = NULL;
+        quota_priv_t    *priv = NULL;
+        int              ret  = -1;
+
+        priv = this->private;
+        if (priv->rpc_clnt) {
+                gf_log (this->name, GF_LOG_TRACE, "quota enforcer clnt already "
+                        "inited");
+                //Turns out to be a NOP if the clnt is already connected.
+                ret = quota_enforcer_blocking_connect (priv->rpc_clnt);
+                if (ret)
+                        goto out;
+
+                return priv->rpc_clnt;
+        }
+
+        priv->quota_enforcer = &quota_enforcer_clnt;
+
+        ret = dict_set_str (options, "transport.address-family", "unix");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (options, "transport-type", "socket");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (options, "transport.socket.connect-path",
+                            "/tmp/quotad.socket");
+        if (ret)
+                goto out;
+
+        rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
+        if (!rpc) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = rpc_clnt_register_notify (rpc, quota_enforcer_notify, this);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "failed to register notify");
+                goto out;
+        }
+
+        ret = quota_enforcer_blocking_connect (rpc);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        if (ret) {
+                if (rpc)
+                        rpc_clnt_unref (rpc);
+                rpc = NULL;
+        }
+
+        return rpc;
+}
+
+struct rpc_clnt_procedure quota_enforcer_actors[GF_AGGREGATOR_MAXVALUE] = {
+        [GF_AGGREGATOR_NULL]     = {"NULL", NULL},
+        [GF_AGGREGATOR_LOOKUP]   = {"LOOKUP", NULL},
+};
+
+struct rpc_clnt_program quota_enforcer_clnt = {
+        .progname  = "Quota enforcer",
+        .prognum   = GLUSTER_AGGREGATOR_PROGRAM,
+        .progver   = GLUSTER_AGGREGATOR_VERSION,
+        .numproc   = GF_AGGREGATOR_MAXVALUE,
+        .proctable = quota_enforcer_actors,
+};
diff --git a/xlators/features/quota/src/quota-mem-types.h b/xlators/features/quota/src/quota-mem-types.h
index ed70c2928..97d916568 100644
--- a/xlators/features/quota/src/quota-mem-types.h
+++ b/xlators/features/quota/src/quota-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __QUOTA_MEM_TYPES_H__
 #define __QUOTA_MEM_TYPES_H__
 
@@ -31,6 +21,9 @@ enum gf_quota_mem_types_ {
         gf_quota_mt_int32_t,
         gf_quota_mt_limits_t,
         gf_quota_mt_quota_dentry_t,
+        gf_quota_mt_quota_limits_level_t,
+        gf_quota_mt_qd_vols_conf_t,
+        gf_quota_mt_aggregator_state_t,
         gf_quota_mt_end
 };
 #endif
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index 844a0981d..4beaae341 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -1,65 +1,111 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <fnmatch.h>
 
 #include "quota.h"
 #include "common-utils.h"
 #include "defaults.h"
+#include "statedump.h"
 
 int32_t
 quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
                    char *name, uuid_t par);
+
+int
+quota_fill_inodectx (xlator_t *this, inode_t *inode, dict_t *dict,
+                     loc_t *loc, struct iatt *buf, int32_t *op_errno);
+
 struct volume_options options[];
 
+static int32_t
+__quota_init_inode_ctx (inode_t *inode, xlator_t *this,
+                        quota_inode_ctx_t **context)
+{
+        int32_t            ret  = -1;
+        quota_inode_ctx_t *ctx  = NULL;
+
+        if (inode == NULL) {
+                goto out;
+        }
+
+        QUOTA_ALLOC_OR_GOTO (ctx, quota_inode_ctx_t, out);
+
+        LOCK_INIT(&ctx->lock);
+
+        if (context != NULL) {
+                *context = ctx;
+        }
+
+        INIT_LIST_HEAD (&ctx->parents);
+
+        ret = __inode_ctx_put (inode, this, (uint64_t )(long)ctx);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "cannot set quota context in inode (gfid:%s)",
+                        uuid_utoa (inode->gfid));
+        }
+out:
+        return ret;
+}
+
+
+static int32_t
+quota_inode_ctx_get (inode_t *inode, xlator_t *this,
+                     quota_inode_ctx_t **ctx, char create_if_absent)
+{
+        int32_t  ret = 0;
+        uint64_t ctx_int;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &ctx_int);
+
+                if ((ret == 0) && (ctx != NULL)) {
+                        *ctx = (quota_inode_ctx_t *) (unsigned long)ctx_int;
+                } else if (create_if_absent) {
+                        ret = __quota_init_inode_ctx (inode, this, ctx);
+                }
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
 int
 quota_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 {
         int ret = -1;
 
-        if (!loc) {
+        if (!loc || (inode == NULL))
                 return ret;
-        }
 
         if (inode) {
                 loc->inode = inode_ref (inode);
+                uuid_copy (loc->gfid, inode->gfid);
         }
 
         if (parent) {
                 loc->parent = inode_ref (parent);
         }
 
-        loc->path = gf_strdup (path);
-        if (!loc->path) {
-                goto loc_wipe;
-        }
+        if (path != NULL) {
+                loc->path = gf_strdup (path);
 
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name) {
-                loc->name++;
-        } else {
-                goto loc_wipe;
+                loc->name = strrchr (loc->path, '/');
+                if (loc->name) {
+                        loc->name++;
+                }
         }
 
         ret = 0;
 
-loc_wipe:
         if (ret < 0) {
                 loc_wipe (loc);
         }
@@ -89,19 +135,17 @@ quota_inode_loc_fill (inode_t *inode, loc_t *loc)
 
         parent = inode_parent (inode, 0, NULL);
         if (!parent) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "cannot find parent for inode (gfid:%s)",
                         uuid_utoa (inode->gfid));
-                goto err;
         }
 
 ignore_parent:
         ret = inode_path (inode, NULL, &resolvedpath);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "cannot construct path for inode (gfid:%s)",
                         uuid_utoa (inode->gfid));
-                goto err;
         }
 
         ret = quota_loc_fill (loc, inode, parent, resolvedpath);
@@ -136,6 +180,7 @@ quota_local_cleanup (xlator_t *this, quota_local_t *local)
         inode_unref (local->inode);
         LOCK_DESTROY (&local->lock);
 
+        mem_put (local);
 out:
         return 0;
 }
@@ -146,8 +191,13 @@ quota_local_new ()
 {
         quota_local_t *local = NULL;
         local = mem_get0 (THIS->local_pool);
-        if (local)
-                LOCK_INIT (&local->lock);
+        if (local == NULL)
+                goto out;
+
+        LOCK_INIT (&local->lock);
+        local->space_available = -1;
+
+out:
         return local;
 }
 
@@ -165,12 +215,15 @@ __quota_dentry_new (quota_inode_ctx_t *ctx, char *name, uuid_t par)
         dentry->name = gf_strdup (name);
         if (dentry->name == NULL) {
                 GF_FREE (dentry);
+                dentry = NULL;
                 goto err;
         }
 
         uuid_copy (dentry->par, par);
 
-        list_add_tail (&dentry->next, &ctx->parents);
+        if (ctx != NULL)
+                list_add_tail (&dentry->next, &ctx->parents);
+
 err:
         return dentry;
 }
@@ -191,18 +244,64 @@ out:
         return;
 }
 
+inline void
+quota_link_count_decrement (quota_local_t *local)
+{
+        call_stub_t *stub       = NULL;
+        int          link_count = -1;
+
+        if (local == NULL)
+                goto out;
+
+        LOCK (&local->lock);
+        {
+                link_count = --local->link_count;
+                if (link_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+out:
+        return;
+}
+
+inline void
+quota_handle_validate_error (quota_local_t *local, int32_t op_ret,
+                             int32_t op_errno)
+{
+        if (local == NULL)
+                goto out;
+
+        LOCK (&local->lock);
+        {
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        /* we abort checking limits on this path to root */
+        quota_link_count_decrement (local);
+out:
+        return;
+}
 
 int32_t
 quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        quota_local_t     *local          = NULL;
-        uint32_t           validate_count = 0, link_count = 0;
-        int32_t            ret            = 0;
-        quota_inode_ctx_t *ctx            = NULL;
-        int64_t           *size           = 0;
-        uint64_t           value          = 0;
-        call_stub_t       *stub           = NULL;
+        quota_local_t     *local      = NULL;
+        int32_t            ret        = 0;
+        quota_inode_ctx_t *ctx        = NULL;
+        int64_t           *size       = 0;
+        uint64_t           value      = 0;
 
         local = frame->local;
 
@@ -214,7 +313,7 @@ quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         GF_ASSERT (frame);
         GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, unwind, op_errno,
                                         EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, dict, unwind, op_errno,
+        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, unwind, op_errno,
                                         EINVAL);
 
         ret = inode_ctx_get (local->validate_loc.inode, this, &value);
@@ -228,7 +327,7 @@ quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto unwind;
         }
 
-        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        ret = dict_get_bin (xdata, QUOTA_SIZE_KEY, (void **) &size);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "size key not present in dict");
@@ -251,25 +350,7 @@ quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 
 unwind:
-        LOCK (&local->lock);
-        {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-
-                validate_count = --local->validate_count;
-                link_count = local->link_count;
-
-                if ((validate_count == 0) && (link_count == 0)) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
-        }
-        UNLOCK (&local->lock);
-
-        if (stub != NULL) {
-                call_resume (stub);
-        }
-
+        quota_handle_validate_error (local, op_ret, op_errno);
         return 0;
 }
 
@@ -296,34 +377,425 @@ quota_timeout (struct timeval *tv, int32_t timeout)
         return timed_out;
 }
 
+inline void
+quota_add_parent (quota_dentry_t *dentry, struct list_head *list)
+{
+        quota_dentry_t *entry = NULL;
+        gf_boolean_t    found = _gf_false;
+
+        if ((dentry == NULL) || (list == NULL)) {
+                goto out;
+        }
+
+        list_for_each_entry (entry, list, next) {
+                if (uuid_compare (dentry->par, entry->par) == 0) {
+                        found = _gf_true;
+                        goto out;
+                }
+        }
+
+        list_add_tail (&dentry->next, list);
+
+out:
+        return;
+}
 
 int32_t
-quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                   char *name, uuid_t par)
+quota_build_ancestry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno,
+                          gf_dirent_t *entries, dict_t *xdata)
 {
-        int32_t               ret            = -1;
-        inode_t              *_inode         = NULL, *parent = NULL;
-        quota_inode_ctx_t    *ctx            = NULL;
-        quota_priv_t         *priv           = NULL;
-        quota_local_t        *local          = NULL;
-        char                  need_validate  = 0, need_unwind = 0;
-        int64_t               delta          = 0;
-        call_stub_t          *stub           = NULL;
-        int32_t               validate_count = 0, link_count = 0;
-        uint64_t              value          = 0;
-        char                  just_validated = 0;
-        uuid_t                trav_uuid      = {0,};
-
-        GF_VALIDATE_OR_GOTO ("quota", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        inode_t           *parent  = NULL, *tmp_parent = NULL;
+        gf_dirent_t       *entry   = NULL;
+        loc_t              loc     = {0, };
+        quota_dentry_t    *dentry  = NULL, *tmp = NULL;
+        quota_inode_ctx_t *ctx     = NULL;
+        struct list_head   parents = {0, };
+        quota_local_t     *local   = NULL;
+
+        INIT_LIST_HEAD (&parents);
 
         local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
+        frame->local = NULL;
+
+        if (op_ret < 0)
+                goto err;
+
+        parent = inode_parent (local->loc.inode, 0, NULL);
+        if (parent == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "parent is NULL");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if ((op_ret > 0) && (entries != NULL)) {
+                list_for_each_entry (entry, &entries->list, list) {
+                        if (__is_root_gfid (entry->inode->gfid)) {
+                                /* The list contains a sub-list for each
+                                 * possible path to the target inode. Each
+                                 * sub-list starts with the root entry of the
+                                 * tree and is followed by the child entries
+                                 * for a particular path to the target entry.
+                                 * The root entry is an implied sub-list
+                                 * delimiter, as it denotes we have started
+                                 * processing a new path. Reset the parent
+                                 * pointer and continue
+                                 */
+
+                                tmp_parent = NULL;
+                        }
+
+                        uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
+
+                        loc.inode = inode_ref (entry->inode);
+                        loc.parent = inode_ref (tmp_parent);
+                        loc.name = entry->d_name;
+
+                        quota_fill_inodectx (this, entry->inode, entry->dict,
+                                             &loc, &entry->d_stat, &op_errno);
+
+                        tmp_parent = entry->inode;
+
+                        loc_wipe (&loc);
+                }
+        }
+
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
+
+        if (ctx != NULL) {
+                LOCK (&ctx->lock);
+                {
+                        list_for_each_entry (dentry, &ctx->parents, next) {
+                                /* we built ancestry for a non-directory */
+                                tmp = __quota_dentry_new (NULL, dentry->name,
+                                                          dentry->par);
+                                quota_add_parent (tmp, &parents);
+
+                                if (list_empty (&tmp->next)) {
+                                        __quota_dentry_free (tmp);
+                                        tmp = NULL;
+                                }
+                        }
+                }
+                UNLOCK (&ctx->lock);
+        }
+
+        if (list_empty (&parents)) {
+                /* we built ancestry for a directory */
+                list_for_each_entry (entry, &entries->list, list) {
+                        if (entry->inode == local->loc.inode)
+                                break;
+                }
+
+                GF_ASSERT (&entry->list != &entries->list);
+
+                tmp = __quota_dentry_new (NULL, entry->d_name, parent->gfid);
+                quota_add_parent (tmp, &parents);
+        }
+
+        local->ancestry_cbk (&parents, local->loc.inode, 0, 0,
+                             local->ancestry_data);
+        goto cleanup;
+
+err:
+        local->ancestry_cbk (NULL, NULL, -1, op_errno, local->ancestry_data);
+
+cleanup:
+        STACK_DESTROY (frame->root);
+        quota_local_cleanup (this, local);
+
+        if (parent != NULL) {
+                inode_unref (parent);
+                parent = NULL;
+        }
+
+        list_for_each_entry_safe (dentry, tmp, &parents, next) {
+                __quota_dentry_free (dentry);
+        }
+
+        return 0;
+}
+
+int32_t
+quota_build_ancestry_open_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               fd_t *fd, dict_t *xdata)
+{
+        dict_t        *xdata_req = NULL;
+        quota_local_t *local     = NULL;
+
+        if (op_ret < 0) {
+                goto err;
+        }
+
+        xdata_req = dict_new ();
+        if (xdata_req == NULL) {
+                op_ret = -ENOMEM;
+                goto err;
+        }
+
+        op_ret = dict_set_int8 (xdata_req, QUOTA_LIMIT_KEY, 1);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                goto err;
+        }
+
+        op_ret = dict_set_int8 (xdata_req, GET_ANCESTRY_DENTRY_KEY, 1);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                goto err;
+        }
+
+        /* This would ask posix layer to construct dentry chain till root */
+        STACK_WIND (frame, quota_build_ancestry_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp, fd, 0, 0, xdata_req);
+
+        op_ret = 0;
+
+err:
+        fd_unref (fd);
+
+        dict_unref (xdata_req);
+
+        if (op_ret < 0) {
+                local = frame->local;
+                frame->local = NULL;
+
+                local->ancestry_cbk (NULL, NULL, -1, op_errno,
+                                     local->ancestry_data);
+                quota_local_cleanup (this, local);
+                STACK_DESTROY (frame->root);
+        }
+
+        return 0;
+}
+
+int
+quota_build_ancestry (inode_t *inode, quota_ancestry_built_t ancestry_cbk,
+                      void *data)
+{
+        loc_t          loc       = {0, };
+        fd_t          *fd        = NULL;
+        quota_local_t *local     = NULL;
+        call_frame_t  *new_frame = NULL;
+        int            op_errno  = EINVAL;
+        xlator_t      *this      = NULL;
+
+        this = THIS;
+
+        loc.inode = inode_ref (inode);
+        uuid_copy (loc.gfid, inode->gfid);
+
+        fd = fd_create (inode, 0);
+
+        new_frame = create_frame (this, this->ctx->pool);
+        if (new_frame == NULL) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        new_frame->root->uid = new_frame->root->gid = 0;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        new_frame->local = local;
+        local->ancestry_cbk = ancestry_cbk;
+        local->ancestry_data = data;
+        local->loc.inode = inode_ref (inode);
+
+        if (IA_ISDIR (inode->ia_type)) {
+                STACK_WIND (new_frame, quota_build_ancestry_open_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->opendir, &loc, fd,
+                            NULL);
+        } else {
+                STACK_WIND (new_frame, quota_build_ancestry_open_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->open, &loc, 0, fd,
+                            NULL);
+        }
+
+        loc_wipe (&loc);
+        return 0;
+
+err:
+        ancestry_cbk (NULL, NULL, -1, op_errno, data);
+
+        fd_unref (fd);
+
+        local = new_frame->local;
+        new_frame->local = NULL;
+
+        if (local != NULL) {
+                quota_local_cleanup (this, local);
+        }
+
+        if (new_frame != NULL) {
+                STACK_DESTROY (new_frame->root);
+        }
+
+        loc_wipe (&loc);
+        return 0;
+}
+
+int
+quota_validate (call_frame_t *frame, inode_t *inode, xlator_t *this,
+                fop_lookup_cbk_t cbk_fn)
+{
+        quota_local_t     *local = NULL;
+        int                ret   = 0;
+        dict_t            *xdata = NULL;
+        quota_priv_t      *priv  = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+        LOCK (&local->lock);
+        {
+                loc_wipe (&local->validate_loc);
+
+                ret = quota_inode_loc_fill (inode, &local->validate_loc);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot fill loc for inode (gfid:%s), hence "
+                                "aborting quota-checks and continuing with fop",
+                                uuid_utoa (inode->gfid));
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (ret < 0) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        xdata = dict_new ();
+        if (xdata == NULL) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        ret = dict_set_int8 (xdata, QUOTA_SIZE_KEY, 1);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "dict set failed");
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        ret = dict_set_str (xdata, "volume-uuid", priv->volume_uuid);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "dict set failed");
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        ret = quota_enforcer_lookup (frame, this, &local->validate_loc, xdata,
+                                     cbk_fn);
+        if (ret < 0) {
+                ret = -ENOTCONN;
+                goto err;
+        }
+
+        ret = 0;
+err:
+        return ret;
+}
+
+void
+quota_check_limit_continuation (struct list_head *parents, inode_t *inode,
+                                int32_t op_ret, int32_t op_errno, void *data)
+{
+        call_frame_t   *frame        = NULL;
+        xlator_t       *this         = NULL;
+        quota_local_t  *local        = NULL;
+        quota_dentry_t *entry        = NULL;
+        inode_t        *parent       = NULL;
+        int             parent_count = 0;
+
+        frame = data;
+        local = frame->local;
+        this = THIS;
+
+        if ((op_ret < 0) || list_empty (parents)) {
+                if (op_ret >= 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Couldn't build ancestry for inode (gfid:%s). "
+                                "Without knowing ancestors till root, quota "
+                                "cannot be enforced. "
+                                "Hence, failing fop with EIO",
+                                uuid_utoa (inode->gfid));
+                        op_errno = EIO;
+                }
+
+                quota_handle_validate_error (local, -1, op_errno);
+                goto out;
+        }
+
+        list_for_each_entry (entry, parents, next) {
+                parent_count++;
+        }
+
+        LOCK (&local->lock);
+        {
+                local->link_count += (parent_count - 1);
+        }
+        UNLOCK (&local->lock);
+
+        list_for_each_entry (entry, parents, next) {
+                parent = inode_find (inode->table, entry->par);
+
+                quota_check_limit (frame, parent, this, NULL, NULL);
+        }
+
+out:
+        return;
+}
+
+int32_t
+quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
+                   char *name, uuid_t par)
+{
+        int32_t            ret                 = -1, op_errno = EINVAL;
+        inode_t           *_inode              = NULL, *parent = NULL;
+        quota_inode_ctx_t *ctx                 = NULL;
+        quota_priv_t      *priv                = NULL;
+        quota_local_t     *local               = NULL;
+        char               need_validate       = 0;
+        gf_boolean_t       hard_limit_exceeded = 0;
+        int64_t            delta               = 0, wouldbe_size = 0;
+        int64_t            space_available     = 0;
+        uint64_t           value               = 0;
+        char               just_validated      = 0;
+        uuid_t             trav_uuid           = {0,};
+        uint32_t           timeout             = 0;
+
+        GF_VALIDATE_OR_GOTO ("quota", this, err);
+        GF_VALIDATE_OR_GOTO (this->name, frame, err);
+        GF_VALIDATE_OR_GOTO (this->name, inode, err);
+
+        local  = frame->local;
+        GF_VALIDATE_OR_GOTO (this->name, local, err);
 
         delta = local->delta;
 
-        GF_VALIDATE_OR_GOTO (this->name, local->stub, out);
+        GF_VALIDATE_OR_GOTO (this->name, local->stub, err);
+        /* Allow all the trusted clients
+         * Don't block the gluster internal processes like rebalance, gsyncd,
+         * self heal etc from the disk quotas.
+         *
+         * Method: Allow all the clients with PID negative. This is by the
+         * assumption that any kernel assigned pid doesn't have the negative
+         * number.
+         */
+        if (0 > frame->root->pid) {
+                ret = 0;
+                quota_link_count_decrement (local);
+                goto done;
+        }
 
         priv = this->private;
 
@@ -336,10 +808,6 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
         {
                 just_validated = local->just_validated;
                 local->just_validated = 0;
-
-                if (just_validated) {
-                        local->validate_count--;
-                }
         }
         UNLOCK (&local->lock);
 
@@ -348,34 +816,68 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
         }
 
         do {
-                if (ctx != NULL) {
+                if (ctx != NULL && (ctx->hard_lim > 0 || ctx->soft_lim > 0)) {
+                        wouldbe_size = ctx->size + delta;
+
                         LOCK (&ctx->lock);
                         {
-                                if (ctx->limit >= 0) {
-                                        if (!just_validated
-                                            && quota_timeout (&ctx->tv,
-                                                              priv->timeout)) {
-                                                need_validate = 1;
-                                        } else if ((ctx->size + delta)
-                                                   >= ctx->limit) {
-                                                local->op_ret = -1;
-                                                local->op_errno = EDQUOT;
-                                                need_unwind = 1;
-                                        }
+                                timeout = priv->soft_timeout;
+
+                                if ((ctx->soft_lim >= 0)
+                                    && (wouldbe_size > ctx->soft_lim)) {
+                                        timeout = priv->hard_timeout;
+                                }
+
+                                if (!just_validated
+                                    && quota_timeout (&ctx->tv, timeout)) {
+                                        need_validate = 1;
+                                } else if (wouldbe_size >= ctx->hard_lim) {
+                                        hard_limit_exceeded = 1;
                                 }
                         }
                         UNLOCK (&ctx->lock);
 
                         if (need_validate) {
-                                goto validate;
-                        }
+                                ret = quota_validate (frame, _inode, this,
+                                                      quota_validate_cbk);
+                                if (ret < 0) {
+                                        op_errno = -ret;
+                                        goto err;
+                                }
 
-                        if (need_unwind) {
                                 break;
                         }
+
+                        if (hard_limit_exceeded) {
+                                local->op_ret = -1;
+                                local->op_errno = EDQUOT;
+
+                                space_available = ctx->hard_lim - ctx->size;
+
+                                if (space_available < 0)
+                                        space_available = 0;
+
+                                if ((local->space_available < 0)
+                                    || (local->space_available
+                                        > space_available)){
+                                        local->space_available
+                                                = space_available;
+
+                                }
+
+                                if (space_available == 0) {
+                                        op_errno = EDQUOT;
+                                        goto err;
+                                }
+                        }
+
+                        /* We log usage only if quota limit is configured on
+                           that inode. */
+                        quota_log_usage (this, ctx, _inode, delta);
                 }
 
                 if (__is_root_gfid (_inode->gfid)) {
+                        quota_link_count_decrement (local);
                         break;
                 }
 
@@ -387,10 +889,15 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
                 }
 
                 if (parent == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot find parent for inode (gfid:%s), hence "
-                                "aborting enforcing quota-limits and continuing"
-                                " with the fop", uuid_utoa (_inode->gfid));
+                        ret = quota_build_ancestry (_inode,
+                                                    quota_check_limit_continuation,
+                                                    frame);
+                        if (ret < 0) {
+                                op_errno = -ret;
+                                goto err;
+                        }
+
+                        break;
                 }
 
                 inode_unref (_inode);
@@ -406,235 +913,105 @@ quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
                 ctx = (quota_inode_ctx_t *)(unsigned long)value;
         } while (1);
 
-        ret = 0;
-
         if (_inode != NULL) {
                 inode_unref (_inode);
+                _inode = NULL;
         }
 
-        LOCK (&local->lock);
-        {
-                validate_count = local->validate_count;
-                link_count = local->link_count;
-                if ((validate_count == 0) && (link_count == 0)) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
-        }
-        UNLOCK (&local->lock);
-
-        if (stub != NULL) {
-                call_resume (stub);
-        }
-
-out:
-        return ret;
-
-validate:
-        LOCK (&local->lock);
-        {
-                loc_wipe (&local->validate_loc);
-
-                if (just_validated) {
-                        local->validate_count--;
-                }
-
-                local->validate_count++;
-                ret = quota_inode_loc_fill (_inode, &local->validate_loc);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot fill loc for inode (gfid:%s), hence "
-                                "aborting quota-checks and continuing with fop",
-                                uuid_utoa (_inode->gfid));
-                        local->validate_count--;
-                }
-        }
-        UNLOCK (&local->lock);
-
-        if (ret < 0) {
-                goto loc_fill_failed;
-        }
+done:
+        return 0;
 
-        STACK_WIND (frame, quota_validate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, &local->validate_loc,
-                    QUOTA_SIZE_KEY);
+err:
+        quota_handle_validate_error (local, -1, op_errno);
 
-loc_fill_failed:
         inode_unref (_inode);
         return 0;
 }
 
-
-int32_t
-quota_get_limit_value (inode_t *inode, xlator_t *this, int64_t *n)
+inline int
+quota_get_limits (xlator_t *this, dict_t *dict, int64_t *hard_lim,
+                  int64_t *soft_lim)
 {
-        int32_t       ret        = 0;
-        char         *path       = NULL;
-        limits_t     *limit_node = NULL;
-        quota_priv_t *priv       = NULL;
-
-        if (inode == NULL || n == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        *n = 0;
+        quota_limit_t *limit            = NULL;
+        quota_priv_t  *priv             = NULL;
+        int64_t        soft_lim_percent = 0, *ptr = NULL;
+        int            ret              = 0;
 
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                ret = -1;
+        if ((this == NULL) || (dict == NULL) || (hard_lim == NULL)
+            || (soft_lim == NULL))
                 goto out;
-        }
 
         priv = this->private;
 
-        list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
-                if (strcmp (limit_node->path, path) == 0) {
-                        *n = limit_node->value;
-                        break;
-                }
-        }
-
-out:
-        return ret;
-}
-
+        ret = dict_get_bin (dict, QUOTA_LIMIT_KEY, (void **) &ptr);
+        limit = (quota_limit_t *)ptr;
 
-static int32_t
-__quota_init_inode_ctx (inode_t *inode, int64_t limit, xlator_t *this,
-                        dict_t *dict, struct iatt *buf,
-                        quota_inode_ctx_t **context)
-{
-        int32_t            ret  = -1;
-        int64_t           *size = 0;
-        quota_inode_ctx_t *ctx  = NULL;
-
-        if (inode == NULL) {
-                goto out;
+        if (limit) {
+                *hard_lim = ntoh64 (limit->hard_lim);
+                soft_lim_percent = ntoh64 (limit->soft_lim_percent);
         }
 
-        QUOTA_ALLOC_OR_GOTO (ctx, quota_inode_ctx_t, out);
-
-        ctx->limit = limit;
-        if (buf)
-                ctx->buf = *buf;
-
-        LOCK_INIT(&ctx->lock);
-
-        if (context != NULL) {
-                *context = ctx;
+        if (soft_lim_percent < 0) {
+                soft_lim_percent = priv->default_soft_lim;
         }
 
-        INIT_LIST_HEAD (&ctx->parents);
-
-        if (dict != NULL) {
-                ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
-                if (ret == 0) {
-                        ctx->size = ntoh64 (*size);
-                        gettimeofday (&ctx->tv, NULL);
-                }
+        if ((*hard_lim > 0) && (soft_lim_percent > 0)) {
+                *soft_lim = (soft_lim_percent * (*hard_lim))/100;
         }
 
-        ret = __inode_ctx_put (inode, this, (uint64_t )(long)ctx);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot set quota context in inode (gfid:%s)",
-                        uuid_utoa (inode->gfid));
-        }
 out:
-        return ret;
-}
-
-
-static int32_t
-quota_inode_ctx_get (inode_t *inode, int64_t limit, xlator_t *this,
-                     dict_t *dict, struct iatt *buf, quota_inode_ctx_t **ctx,
-                     char create_if_absent)
-{
-        int32_t  ret = 0;
-        uint64_t ctx_int;
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_int);
-
-                if ((ret == 0) && (ctx != NULL)) {
-                        *ctx = (quota_inode_ctx_t *) (unsigned long)ctx_int;
-                } else if (create_if_absent) {
-                        ret = __quota_init_inode_ctx (inode, limit, this, dict,
-                                                      buf, ctx);
-                }
-        }
-        UNLOCK (&inode->lock);
-
-        return ret;
+        return 0;
 }
 
-
-int32_t
-quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, dict_t *dict, struct iatt *postparent)
+int
+quota_fill_inodectx (xlator_t *this, inode_t *inode, dict_t *dict,
+                     loc_t *loc, struct iatt *buf, int32_t *op_errno)
 {
-        int32_t            ret    = -1;
-        char               found  = 0;
-        quota_local_t     *local  = NULL;
-        quota_inode_ctx_t *ctx    = NULL;
-        quota_dentry_t    *dentry = NULL;
-        int64_t           *size   = 0;
-        uint64_t           value  = 0;
+        int32_t            ret      = -1;
+        char               found    = 0;
+        quota_inode_ctx_t *ctx      = NULL;
+        quota_dentry_t    *dentry   = NULL;
+        uint64_t           value    = 0;
+        int64_t            hard_lim = -1, soft_lim = -1;
 
-        local = frame->local;
+        quota_get_limits (this, dict, &hard_lim, &soft_lim);
 
         inode_ctx_get (inode, this, &value);
         ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
-        if ((op_ret < 0) || (local == NULL)
-            || (((ctx == NULL) || (ctx->limit == local->limit))
-                && (local->limit < 0) && !((IA_ISREG (buf->ia_type))
-                                           || (IA_ISLNK (buf->ia_type))))) {
-                goto unwind;
+        if ((((ctx == NULL) || (ctx->hard_lim == hard_lim))
+             && (hard_lim < 0) && !QUOTA_REG_OR_LNK_FILE (buf->ia_type))) {
+                ret = 0;
+                goto out;
         }
 
-        if (local->loc.name == NULL)
-                goto unwind;
-
-        ret = quota_inode_ctx_get (local->loc.inode, local->limit, this, dict,
-                                   buf, &ctx, 1);
+        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
         if ((ret == -1) || (ctx == NULL)) {
                 gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
                         "context in inode(gfid:%s)",
-                        uuid_utoa (local->loc.inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
+                        uuid_utoa (inode->gfid));
+                ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
         }
 
         LOCK (&ctx->lock);
         {
-
-                if (dict != NULL) {
-                        ret = dict_get_bin (dict, QUOTA_SIZE_KEY,
-                                            (void **) &size);
-                        if (ret == 0) {
-                                ctx->size = ntoh64 (*size);
-                                gettimeofday (&ctx->tv, NULL);
-                        }
-                }
-
-                if (local->limit != ctx->limit) {
-                        ctx->limit = local->limit;
-                }
+                ctx->hard_lim = hard_lim;
+                ctx->soft_lim = soft_lim;
 
                 ctx->buf = *buf;
 
-                if (!(IA_ISREG (buf->ia_type) || IA_ISLNK (buf->ia_type))) {
+                if (!QUOTA_REG_OR_LNK_FILE (buf->ia_type)) {
                         goto unlock;
                 }
 
+                if (loc->name == NULL)
+                        goto unlock;
+
                 list_for_each_entry (dentry, &ctx->parents, next) {
-                        if ((strcmp (dentry->name, local->loc.name) == 0) &&
-                            (uuid_compare (local->loc.parent->gfid,
+                        if ((strcmp (dentry->name, loc->name) == 0) &&
+                            (uuid_compare (loc->parent->gfid,
                                            dentry->par) == 0)) {
                                 found = 1;
                                 break;
@@ -643,18 +1020,18 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 if (!found) {
                         dentry = __quota_dentry_new (ctx,
-                                                     (char *)local->loc.name,
-                                                     local->loc.parent->gfid);
+                                                     (char *)loc->name,
+                                                     loc->parent->gfid);
                         if (dentry == NULL) {
                                 /*
-                                gf_log (this->name, GF_LOG_WARNING,
+                                  gf_log (this->name, GF_LOG_WARNING,
                                         "cannot create a new dentry (par:%"
                                         PRId64", name:%s) for inode(ino:%"
                                         PRId64", gfid:%s)",
                                         uuid_utoa (local->loc.inode->gfid));
                                 */
-                                op_ret = -1;
-                                op_errno = ENOMEM;
+                                ret = -1;
+                                *op_errno = ENOMEM;
                                 goto unlock;
                         }
                 }
@@ -662,6 +1039,25 @@ quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unlock:
         UNLOCK (&ctx->lock);
 
+out:
+        return ret;
+}
+
+int32_t
+quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *dict, struct iatt *postparent)
+{
+        quota_local_t *local = NULL;
+
+        if (op_ret < 0)
+                goto unwind;
+
+        local = frame->local;
+
+        op_ret = quota_fill_inodectx (this, inode, dict, &local->loc, buf,
+                                      &op_errno);
+
 unwind:
         QUOTA_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
                             dict, postparent);
@@ -673,144 +1069,64 @@ int32_t
 quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
               dict_t *xattr_req)
 {
-        int32_t             ret         = -1;
-        int64_t             limit       = -1;
-        limits_t           *limit_node  = NULL;
-        gf_boolean_t        dict_newed  = _gf_false;
-        quota_priv_t       *priv        = NULL;
-        quota_local_t      *local       = NULL;
+        quota_priv_t  *priv             = NULL;
+        int32_t        ret              = -1;
+        quota_local_t *local            = NULL;
 
         priv = this->private;
 
-        list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
-                if (strcmp (limit_node->path, loc->path) == 0) {
-                        limit = limit_node->value;
-                }
-        }
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
+        xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+        if (!xattr_req)
                 goto err;
-        }
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1) {
+        local = quota_local_new ();
+        if (local == NULL) {
                 goto err;
         }
 
         frame->local = local;
+        loc_copy (&local->loc, loc);
 
-        local->limit = limit;
-
-        if (limit < 0) {
-                goto wind;
-        }
-
-        if (xattr_req == NULL) {
-                xattr_req  = dict_new ();
-                dict_newed = _gf_true;
-        }
-
-        ret = dict_set_uint64 (xattr_req, QUOTA_SIZE_KEY, 0);
+        ret = dict_set_int8 (xattr_req, QUOTA_LIMIT_KEY, 1);
         if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict set of key for hard-limit failed");
                 goto err;
         }
 
-wind:
         STACK_WIND (frame, quota_lookup_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
 
         ret = 0;
 
 err:
+        if (xattr_req)
+                dict_unref (xattr_req);
+
         if (ret < 0) {
                 QUOTA_STACK_UNWIND (lookup, frame, -1, ENOMEM,
                                     NULL, NULL, NULL, NULL);
         }
 
-        if (dict_newed == _gf_true) {
-                dict_unref (xattr_req);
-        }
-
         return 0;
-}
-
-
-void
-quota_update_size (xlator_t *this, inode_t *inode, char *name, uuid_t par,
-                   int64_t delta)
-{
-        inode_t           *_inode    = NULL;
-        inode_t           *parent    = NULL;
-        uint64_t           value     = 0;
-        quota_inode_ctx_t *ctx       = NULL;
-        uuid_t             trav_uuid = {0,};
-
-        GF_VALIDATE_OR_GOTO ("quota", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        inode_ctx_get (inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
-        _inode = inode_ref (inode);
-
-        if ( par != NULL ) {
-                uuid_copy (trav_uuid, par);
-        }
-
-        do {
-                if ((ctx != NULL) && (ctx->limit >= 0)) {
-                        LOCK (&ctx->lock);
-                        {
-                                ctx->size += delta;
-                        }
-                        UNLOCK (&ctx->lock);
-                }
-
-                if (__is_root_gfid (_inode->gfid)) {
-                        break;
-                }
-
-                parent = inode_parent (_inode, trav_uuid, name);
-                if (parent == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot find parent for inode (gfid:%s), hence "
-                                "aborting size updation of parents",
-                                uuid_utoa (_inode->gfid));
-                }
-
-                if (name != NULL) {
-                        name = NULL;
-                        uuid_clear (trav_uuid);
-                }
-
-                inode_unref (_inode);
-                _inode = parent;
-
-                if (_inode == NULL) {
-                        break;
-                }
-
-                inode_ctx_get (_inode, this, &value);
-                ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        } while (1);
-
-out:
-        return;
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+        return 0;
 }
 
-
 int32_t
 quota_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
         int32_t                  ret            = 0;
         uint64_t                 ctx_int        = 0;
         quota_inode_ctx_t       *ctx            = NULL;
         quota_local_t           *local          = NULL;
-        quota_dentry_t          *dentry         = NULL;
-        int64_t                  delta          = 0;
 
         local = frame->local;
 
@@ -840,14 +1156,9 @@ quota_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
         UNLOCK (&ctx->lock);
 
-        list_for_each_entry (dentry, &ctx->parents, next) {
-                delta = (postbuf->ia_blocks - prebuf->ia_blocks) * 512;
-                quota_update_size (this, local->loc.inode,
-                                   dentry->name, dentry->par, delta);
-        }
-
 out:
-        QUOTA_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        QUOTA_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
 
         return 0;
 }
@@ -856,10 +1167,15 @@ out:
 int32_t
 quota_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
                      struct iovec *vector, int32_t count, off_t off,
-                     uint32_t flags, struct iobref *iobref)
+                     uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
+        quota_local_t *local      = NULL;
+        int32_t        op_errno   = EINVAL;
+        quota_priv_t  *priv       = NULL;
+        struct iovec  *new_vector = NULL;
+        int32_t        new_count  = 0;
+
+        priv = this->private;
 
         local = frame->local;
         if (local == NULL) {
@@ -869,16 +1185,42 @@ quota_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         if (local->op_ret == -1) {
                 op_errno = local->op_errno;
-                goto unwind;
+
+                if ((op_errno == EDQUOT) && (local->space_available > 0)) {
+                        new_count = iov_subset (vector, count, 0,
+                                                local->space_available, NULL);
+
+                        new_vector = GF_CALLOC (new_count,
+                                                sizeof (struct iovec),
+                                                gf_common_mt_iovec);
+                        if (new_vector == NULL) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unwind;
+                        }
+
+                        new_count = iov_subset (vector, count, 0,
+                                                local->space_available,
+                                                new_vector);
+
+                        vector = new_vector;
+                        count = new_count;
+                } else {
+                        goto unwind;
+                }
         }
 
-        STACK_WIND (frame, quota_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
-                    flags, iobref);
+        STACK_WIND (frame, quota_writev_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev, fd,
+                    vector, count, off, flags, iobref, xdata);
+
+        if (new_vector != NULL)
+                GF_FREE (new_vector);
+
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -886,16 +1228,23 @@ unwind:
 int32_t
 quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
               struct iovec *vector, int32_t count, off_t off,
-              uint32_t flags, struct iobref *iobref)
+              uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
+        quota_priv_t      *priv    = NULL;
         int32_t            ret     = -1, op_errno = EINVAL;
         int32_t            parents = 0;
         uint64_t           size    = 0;
         quota_local_t     *local   = NULL;
         quota_inode_ctx_t *ctx     = NULL;
-        quota_priv_t      *priv    = NULL;
+        quota_dentry_t    *dentry  = NULL, *tmp = NULL;
         call_stub_t       *stub    = NULL;
-        quota_dentry_t    *dentry  = NULL;
+        struct list_head   head    = {0, };
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
+        INIT_LIST_HEAD (&head);
 
         GF_ASSERT (frame);
         GF_VALIDATE_OR_GOTO ("quota", this, unwind);
@@ -909,16 +1258,17 @@ quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         frame->local = local;
         local->loc.inode = inode_ref (fd->inode);
 
-        ret = quota_inode_ctx_get (fd->inode, -1, this, NULL, NULL, &ctx, 0);
+        ret = quota_inode_ctx_get (fd->inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (fd->inode->gfid));
-                goto unwind;
         }
 
         stub = fop_writev_stub (frame, quota_writev_helper, fd, vector, count,
-                                off, flags, iobref);
+                                off, flags, iobref, xdata);
         if (stub == NULL) {
                 op_errno = ENOMEM;
                 goto unwind;
@@ -928,46 +1278,50 @@ quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
 
         size = iov_length (vector, count);
-        LOCK (&ctx->lock);
-        {
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        parents++;
-                }
-        }
-        UNLOCK (&ctx->lock);
-
-        local->delta = size;
-        local->stub = stub;
-        local->link_count = parents;
-
-        list_for_each_entry (dentry, &ctx->parents, next) {
-                ret = quota_check_limit (frame, fd->inode, this, dentry->name,
-                                         dentry->par);
-                if (ret == -1) {
-                        break;
+        if (ctx != NULL) {
+                LOCK (&ctx->lock);
+                {
+                        list_for_each_entry (dentry, &ctx->parents, next) {
+                                tmp = __quota_dentry_new (NULL, dentry->name,
+                                                          dentry->par);
+                                list_add_tail (&tmp->next, &head);
+                                parents++;
+                        }
                 }
+                UNLOCK (&ctx->lock);
         }
 
-        stub = NULL;
-
         LOCK (&local->lock);
         {
-                local->link_count = 0;
-                if (local->validate_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
+                local->delta = size;
+                local->link_count = (parents != 0) ? parents : 1;
+                local->stub = stub;
         }
         UNLOCK (&local->lock);
 
-        if (stub != NULL) {
-                call_resume (stub);
+        if (parents == 0) {
+                /* nameless lookup on this inode, allow quota to reconstruct
+                 * ancestry as part of check_limit.
+                 */
+                quota_check_limit (frame, fd->inode, this, NULL, NULL);
+        } else {
+                list_for_each_entry_safe (dentry, tmp, &head, next) {
+                        quota_check_limit (frame, fd->inode, this, dentry->name,
+                                           dentry->par);
+                        __quota_dentry_free (dentry);
+                }
         }
 
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->writev, fd,
+                         vector, count, off, flags, iobref, xdata);
         return 0;
 }
 
@@ -976,17 +1330,17 @@ int32_t
 quota_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         QUOTA_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode,
-                            buf, preparent, postparent);
+                            buf, preparent, postparent, xdata);
         return 0;
 }
 
 
 int32_t
 quota_mkdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    mode_t mode, dict_t *params)
+                    mode_t mode, mode_t umask, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
@@ -1003,24 +1357,31 @@ quota_mkdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto unwind;
         }
 
-        STACK_WIND (frame, quota_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
+        STACK_WIND (frame, quota_mkdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc,
+                    mode, umask, xdata);
+
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL);
+                            NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dict_t *params)
+             mode_t umask, dict_t *xdata)
 {
-        int32_t        ret            = 0, op_errno = 0;
-        quota_local_t *local          = NULL;
-        call_stub_t   *stub           = NULL;
+        quota_priv_t  *priv  = NULL;
+        int32_t        ret   = 0, op_errno = 0;
+        quota_local_t *local = NULL;
+        call_stub_t   *stub  = NULL;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
         local = quota_local_new ();
         if (local == NULL) {
@@ -1030,8 +1391,6 @@ quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 
         frame->local = local;
 
-        local->link_count = 1;
-
         ret = loc_copy (&local->loc, loc);
         if (ret) {
                 op_errno = ENOMEM;
@@ -1039,38 +1398,34 @@ quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 goto err;
         }
 
-        stub = fop_mkdir_stub (frame, quota_mkdir_helper, loc, mode, params);
+        stub = fop_mkdir_stub (frame, quota_mkdir_helper, loc, mode, umask,
+                               xdata);
         if (stub == NULL) {
                 op_errno = ENOMEM;
                 goto err;
         }
 
-        local->stub = stub;
-        local->delta = 0;
-
-        quota_check_limit (frame, loc->parent, this, NULL, NULL);
-
-        stub = NULL;
-
         LOCK (&local->lock);
         {
-                if (local->validate_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
-
-                local->link_count = 0;
+                local->stub = stub;
+                local->delta = 0;
+                local->link_count = 1;
         }
         UNLOCK (&local->lock);
 
-        if (stub != NULL) {
-                call_resume (stub);
-        }
-
+        quota_check_limit (frame, loc->parent, this, NULL, NULL);
         return 0;
+
 err:
         QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL);
+                            NULL, NULL);
+
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->mkdir,
+                         loc, mode, umask, xdata);
 
         return 0;
 }
@@ -1080,7 +1435,7 @@ int32_t
 quota_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         int32_t            ret    = -1;
         quota_local_t     *local  = NULL;
@@ -1092,7 +1447,7 @@ quota_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto unwind;
         }
 
-        ret = quota_inode_ctx_get (inode, -1, this, NULL, buf, &ctx, 1);
+        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
         if ((ret == -1) || (ctx == NULL)) {
                 gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
                         "context in inode(gfid:%s)",
@@ -1123,19 +1478,24 @@ unlock:
 
 unwind:
         QUOTA_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                            preparent, postparent);
+                            preparent, postparent, xdata);
         return 0;
 }
 
 
 int32_t
 quota_create_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+                     int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                     dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
+        quota_priv_t  *priv     = NULL;
 
         local = frame->local;
+
+        priv = this->private;
+
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_WARNING, "local is NULL");
                 goto unwind;
@@ -1146,28 +1506,36 @@ quota_create_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto unwind;
         }
 
-        STACK_WIND (frame, quota_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, fd,
-                    params);
+
+        STACK_WIND (frame, quota_create_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create, loc,
+                    flags, mode, umask, fd, xdata);
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
+                            NULL, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              mode_t mode, fd_t *fd, dict_t *params)
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int32_t            ret            = -1;
-        quota_local_t     *local          = NULL;
-        call_stub_t       *stub           = NULL;
+        quota_priv_t  *priv     = NULL;
+        int32_t        ret      = -1;
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = 0;
+        call_stub_t   *stub     = NULL;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
         local = quota_local_new ();
         if (local == NULL) {
+                op_errno = ENOMEM;
                 goto err;
         }
 
@@ -1176,42 +1544,36 @@ quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         ret = loc_copy (&local->loc, loc);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                op_errno = ENOMEM;
                 goto err;
         }
 
         stub = fop_create_stub (frame, quota_create_helper, loc, flags, mode,
-                                fd, params);
+                                umask, fd, xdata);
         if (stub == NULL) {
                 goto err;
         }
 
-        local->link_count = 1;
-        local->stub = stub;
-        local->delta = 0;
-
-        quota_check_limit (frame, loc->parent, this, NULL, NULL);
-
-        stub = NULL;
-
         LOCK (&local->lock);
         {
-                local->link_count = 0;
-                if (local->validate_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
+                local->link_count = 1;
+                local->stub = stub;
+                local->delta = 0;
         }
         UNLOCK (&local->lock);
 
-        if (stub != NULL) {
-                call_resume (stub);
-        }
-
+        quota_check_limit (frame, loc->parent, this, NULL, NULL);
         return 0;
 err:
-        QUOTA_STACK_UNWIND (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                            NULL);
+        QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL, NULL, NULL);
+
+        return 0;
 
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
+                         FIRST_CHILD (this)->fops->create, loc,
+                         flags, mode, umask, fd, xdata);
         return 0;
 }
 
@@ -1219,11 +1581,13 @@ err:
 int32_t
 quota_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
         uint64_t           value = 0;
+        quota_dentry_t    *dentry = NULL;
+        quota_dentry_t    *old_dentry = NULL;
 
         if (op_ret < 0) {
                 goto out;
@@ -1241,23 +1605,40 @@ quota_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        quota_update_size (this, local->loc.inode, (char *)local->loc.name,
-                           local->loc.parent->gfid,
-                           (-(ctx->buf.ia_blocks * 512)));
+        LOCK (&ctx->lock);
+        {
+                list_for_each_entry (dentry, &ctx->parents, next) {
+                        if ((strcmp (dentry->name, local->loc.name) == 0) &&
+                            (uuid_compare (local->loc.parent->gfid,
+                                           dentry->par) == 0)) {
+                                old_dentry = dentry;
+                                break;
+                        }
+                }
+                if (old_dentry)
+                        __quota_dentry_free (old_dentry);
+        }
+        UNLOCK (&ctx->lock);
 
 out:
         QUOTA_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent,
-                            postparent);
+                            postparent, xdata);
         return 0;
 }
 
 
 int32_t
-quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
-        int32_t        ret = 0;
+        quota_priv_t       *priv        = NULL;
+        int32_t        ret = -1;
         quota_local_t *local = NULL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto err;
@@ -1265,6 +1646,10 @@ quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
         frame->local = local;
 
+        if (xdata && dict_get (xdata, GLUSTERFS_INTERNAL_FOP_KEY)) {
+                local->skip_check = _gf_true;
+        }
+
         ret = loc_copy (&local->loc, loc);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
@@ -1272,16 +1657,21 @@ quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         STACK_WIND (frame, quota_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
 
         ret = 0;
 
 err:
         if (ret == -1) {
-                QUOTA_STACK_UNWIND (unlink, frame, -1, 0, NULL, NULL);
+                QUOTA_STACK_UNWIND (unlink, frame, -1, 0, NULL, NULL, NULL);
         }
 
         return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
 }
 
 
@@ -1289,7 +1679,7 @@ int32_t
 quota_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, inode_t *inode,
                 struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
         int32_t               ret          = -1;
         quota_local_t        *local        = NULL;
@@ -1303,16 +1693,16 @@ quota_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local = (quota_local_t *) frame->local;
 
-        quota_update_size (this, local->loc.parent, NULL, NULL,
-                           (buf->ia_blocks * 512));
+        if (local->skip_check)
+                goto out;
 
-        ret = quota_inode_ctx_get (inode, -1, this, NULL, NULL, &ctx, 0);
+        ret = quota_inode_ctx_get (inode, this, &ctx, 0);
         if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_WARNING, "cannot find quota "
-                        "context in %s (gfid:%s)", local->loc.path,
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
                 goto out;
         }
 
@@ -1354,7 +1744,7 @@ unlock:
 
 out:
         QUOTA_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent);
+                            preparent, postparent, xdata);
 
         return 0;
 }
@@ -1362,10 +1752,13 @@ out:
 
 int32_t
 quota_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                   loc_t *newloc)
+                   loc_t *newloc, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
+        quota_priv_t  *priv     = NULL;
+
+        priv = this->private;
 
         local = frame->local;
         if (local == NULL) {
@@ -1379,24 +1772,44 @@ quota_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                 goto unwind;
         }
 
-        STACK_WIND (frame, quota_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
+        STACK_WIND (frame, quota_link_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link, oldloc,
+                    newloc, xdata);
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL);
+                            NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
-quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
+        quota_priv_t      *priv  = NULL;
         int32_t            ret   = -1, op_errno = ENOMEM;
         quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
         call_stub_t       *stub  = NULL;
-        quota_inode_ctx_t *ctx = NULL;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
+        if (xdata && dict_get (xdata, GLUSTERFS_INTERNAL_FOP_KEY)) {
+                goto off;
+        }
+
+        quota_inode_ctx_get (oldloc->inode, this, &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
+                        uuid_utoa (oldloc->inode->gfid));
+        }
 
         local = quota_local_new ();
         if (local == NULL) {
@@ -1405,58 +1818,38 @@ quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
 
         frame->local = (void *) local;
 
+
         ret = loc_copy (&local->loc, newloc);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
                 goto err;
         }
 
-        stub = fop_link_stub (frame, quota_link_helper, oldloc, newloc);
+        stub = fop_link_stub (frame, quota_link_helper, oldloc, newloc, xdata);
         if (stub == NULL) {
                 goto err;
         }
 
-        local->link_count = 1;
-        local->stub = stub;
-
-        ret = quota_inode_ctx_get (oldloc->inode, -1, this, NULL, NULL, &ctx,
-                                   0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
-                        oldloc->inode ? uuid_utoa (oldloc->inode->gfid) : "0");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        local->delta = ctx->buf.ia_blocks * 512;
-
-        quota_check_limit (frame, newloc->parent, this, NULL, NULL);
-
-        stub = NULL;
-
         LOCK (&local->lock);
         {
-                if (local->validate_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
-
-                local->link_count = 0;
+                local->link_count = 1;
+                local->stub = stub;
+                local->delta = (ctx != NULL) ? ctx->buf.ia_blocks * 512 : 0;
         }
         UNLOCK (&local->lock);
 
-        if (stub != NULL) {
-                call_resume (stub);
-        }
+        quota_check_limit (frame, newloc->parent, this, NULL, NULL);
+        return 0;
 
-        ret = 0;
 err:
-        if (ret < 0) {
-                QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                                    NULL, NULL);
-        }
+        QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
+                            NULL, NULL, NULL);
+        return 0;
 
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->link, oldloc,
+                         newloc, xdata);
         return 0;
 }
 
@@ -1465,14 +1858,15 @@ int32_t
 quota_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
                   struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent)
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
         int32_t               ret              = -1;
+        int64_t               size             = 0;
         quota_local_t        *local            = NULL;
         quota_inode_ctx_t    *ctx              = NULL;
         quota_dentry_t       *old_dentry       = NULL, *dentry = NULL;
         char                  new_dentry_found = 0;
-        int64_t               size             = 0;
 
         if (op_ret < 0) {
                 goto out;
@@ -1486,29 +1880,22 @@ quota_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        if (IA_ISREG (local->oldloc.inode->ia_type)
-            || IA_ISLNK (local->oldloc.inode->ia_type)) {
+        if (QUOTA_REG_OR_LNK_FILE (local->oldloc.inode->ia_type)) {
                 size = buf->ia_blocks * 512;
         }
 
-        if (local->oldloc.parent != local->newloc.parent) {
-                quota_update_size (this, local->oldloc.parent, NULL, NULL, (-size));
-                quota_update_size (this, local->newloc.parent, NULL, NULL, size);
-        }
-
-        if (!(IA_ISREG (local->oldloc.inode->ia_type)
-              || IA_ISLNK (local->oldloc.inode->ia_type))) {
+        if (!QUOTA_REG_OR_LNK_FILE (local->oldloc.inode->ia_type)) {
                 goto out;
         }
 
-        ret = quota_inode_ctx_get (local->oldloc.inode, -1, this, NULL, NULL,
-                                   &ctx, 0);
+        ret = quota_inode_ctx_get (local->oldloc.inode, this, &ctx, 0);
         if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_WARNING, "quota context not"
-                        "set in inode(gfid:%s)",
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (local->oldloc.inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
+
                 goto out;
         }
 
@@ -1554,7 +1941,8 @@ quota_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         if (dentry == NULL) {
                                 gf_log (this->name, GF_LOG_WARNING,
                                         "cannot create a new dentry (name:%s) "
-                                        "for inode(gfid:%s)", local->newloc.name,
+                                        "for inode(gfid:%s)",
+                                        local->newloc.name,
                                         uuid_utoa (local->newloc.inode->gfid));
                                 op_ret = -1;
                                 op_errno = ENOMEM;
@@ -1569,7 +1957,7 @@ unlock:
 
 out:
         QUOTA_STACK_UNWIND (rename, frame, op_ret, op_errno, buf, preoldparent,
-                            postoldparent, prenewparent, postnewparent);
+                            postoldparent, prenewparent, postnewparent, xdata);
 
         return 0;
 }
@@ -1577,10 +1965,13 @@ out:
 
 int32_t
 quota_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                     loc_t *newloc)
+                     loc_t *newloc, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
+        quota_priv_t  *priv     = NULL;
+
+        priv = this->private;
 
         local = frame->local;
         if (local == NULL) {
@@ -1594,25 +1985,72 @@ quota_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                 goto unwind;
         }
 
-        STACK_WIND (frame, quota_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
+        STACK_WIND (frame, quota_rename_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename, oldloc,
+                    newloc, xdata);
+
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
+                            NULL, NULL, NULL, NULL);
         return 0;
 }
 
 
+static int32_t
+quota_rename_get_size_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
+{
+        quota_local_t     *local      = NULL;
+        int32_t            ret        = 0;
+        int64_t           *size       = 0;
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, out, op_errno,
+                                        EINVAL);
+        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, out, op_errno,
+                                        EINVAL);
+        local = frame->local;
+        GF_ASSERT (local);
+        local->link_count = 1;
+
+        if (op_ret < 0)
+                goto out;
+
+
+        ret = dict_get_bin (xdata, QUOTA_SIZE_KEY, (void **) &size);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "size key not present in dict");
+                op_errno = EINVAL;
+                goto out;
+        }
+        local->delta = ntoh64 (*size);
+        quota_check_limit (frame, local->newloc.parent, this,
+                           NULL, NULL);
+        return 0;
+
+out:
+        quota_handle_validate_error (local, -1, op_errno);
+        return 0;
+}
+
 int32_t
 quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-              loc_t *newloc)
+              loc_t *newloc, dict_t *xdata)
 {
-        int32_t            ret            = -1, op_errno = ENOMEM;
-        quota_local_t     *local          = NULL;
-        call_stub_t       *stub           = NULL;
-        quota_inode_ctx_t *ctx            = NULL;
+        quota_priv_t      *priv  = NULL;
+        int32_t            ret   = -1, op_errno = ENOMEM;
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+        call_stub_t       *stub  = NULL;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
         local = quota_local_new ();
         if (local == NULL) {
@@ -1633,56 +2071,69 @@ quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                 goto err;
         }
 
-        stub = fop_rename_stub (frame, quota_rename_helper, oldloc, newloc);
+        stub = fop_rename_stub (frame, quota_rename_helper, oldloc, newloc,
+                                xdata);
         if (stub == NULL) {
                 goto err;
         }
 
-        local->link_count = 1;
-        local->stub = stub;
+        LOCK (&local->lock);
+        {
+                local->link_count = 1;
+                local->stub = stub;
+        }
+        UNLOCK (&local->lock);
 
-        if (IA_ISREG (oldloc->inode->ia_type)
-            || IA_ISLNK (oldloc->inode->ia_type)) {
-                ret = quota_inode_ctx_get (oldloc->inode, -1, this, NULL, NULL,
-                                           &ctx, 0);
+        if (QUOTA_REG_OR_LNK_FILE (oldloc->inode->ia_type)) {
+                ret = quota_inode_ctx_get (oldloc->inode, this, &ctx, 0);
                 if (ctx == NULL) {
                         gf_log (this->name, GF_LOG_WARNING,
-                                "quota context not set in inode (gfid:%s)",
+                                "quota context not set in inode (gfid:%s), "
+                                "considering file size as zero while enforcing "
+                                "quota on new ancestry",
                                 oldloc->inode ? uuid_utoa (oldloc->inode->gfid)
                                 : "0");
-                        op_errno = EINVAL;
-                        goto err;
-                }
-                local->delta = ctx->buf.ia_blocks * 512;
-        } else {
-                local->delta = 0;
-        }
+                        local->delta = 0;
 
-        quota_check_limit (frame, newloc->parent, this, NULL, NULL);
+                } else {
 
-        stub = NULL;
+                    /* FIXME: We need to account for the size occupied by this
+                     * inode on the target directory. To avoid double
+                     * accounting, we need to modify enforcer to perform
+                     * quota_check_limit only uptil the least common ancestor
+                     * directory inode*/
 
-        LOCK (&local->lock);
-        {
-                if (local->validate_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
+                    /* FIXME: The following code assumes that regular files and
+                     *linkfiles are present, in their entirety, in a single
+                     brick. This *assumption is invalid in the case of
+                     stripe.*/
+
+                    local->delta = ctx->buf.ia_blocks * 512;
                 }
 
-                local->link_count = 0;
-        }
-        UNLOCK (&local->lock);
+        } else if (IA_ISDIR (oldloc->inode->ia_type)) {
+                        ret = quota_validate (frame, oldloc->inode, this,
+                                              quota_rename_get_size_cbk);
+                        if (ret){
+                                op_errno = -ret;
+                                goto err;
+                        }
 
-        if (stub != NULL) {
-                call_resume (stub);
+                        return 0;
         }
 
-        ret = 0;
+        quota_check_limit (frame, newloc->parent, this, NULL, NULL);
+        return 0;
+
 err:
-        if (ret == -1) {
-                QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
-                                    NULL, NULL, NULL, NULL);
-        }
+        QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
+                            NULL, NULL, NULL, NULL, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rename, oldloc,
+                         newloc, xdata);
 
         return 0;
 }
@@ -1692,9 +2143,8 @@ int32_t
 quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
-        int64_t            size   = 0;
         quota_local_t     *local  = NULL;
         quota_inode_ctx_t *ctx    = NULL;
         quota_dentry_t    *dentry = NULL;
@@ -1704,16 +2154,15 @@ quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         local = frame->local;
-        size = buf->ia_blocks * 512;
-
-        quota_update_size (this, local->loc.parent, NULL, NULL, size);
 
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 1);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (local->loc.inode->gfid));
+
                 goto out;
         }
 
@@ -1736,7 +2185,7 @@ quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent);
+                            preparent, postparent, xdata);
 
         return 0;
 }
@@ -1744,10 +2193,11 @@ out:
 
 int
 quota_symlink_helper (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                      loc_t *loc, dict_t *params)
+                      loc_t *loc, mode_t umask, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
+        quota_priv_t  *priv     = NULL;
 
         local = frame->local;
         if (local == NULL) {
@@ -1755,30 +2205,38 @@ quota_symlink_helper (call_frame_t *frame, xlator_t *this, const char *linkpath,
                 goto unwind;
         }
 
+        priv = this->private;
+
         if (local->op_ret == -1) {
                 op_errno = local->op_errno;
                 goto unwind;
         }
 
-        STACK_WIND (frame, quota_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, params);
+        STACK_WIND (frame, quota_symlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkpath, loc, umask, xdata);
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL);
+                            NULL, NULL, NULL);
         return 0;
 }
 
 
 int
 quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, dict_t *params)
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int32_t          ret      = -1;
-        int32_t          op_errno = ENOMEM;
-        quota_local_t   *local    = NULL;
-        call_stub_t     *stub     = NULL;
+        quota_priv_t  *priv     = NULL;
+        int32_t        ret      = -1;
+        int32_t        op_errno = ENOMEM;
+        quota_local_t *local    = NULL;
+        call_stub_t   *stub     = NULL;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
         local = quota_local_new ();
         if (local == NULL) {
@@ -1793,42 +2251,33 @@ quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
                 goto err;
         }
 
-        local->link_count = 1;
-
         stub = fop_symlink_stub (frame, quota_symlink_helper, linkpath, loc,
-                                 params);
+                                 umask, xdata);
         if (stub == NULL) {
                 goto err;
         }
 
-        local->stub = stub;
-        local->delta = strlen (linkpath);
-
-        quota_check_limit (frame, loc->parent, this, NULL, NULL);
-
-        stub = NULL;
-
         LOCK (&local->lock);
         {
-                if (local->validate_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
-
-                local->link_count = 0;
+                local->stub = stub;
+                local->delta = strlen (linkpath);
+                local->link_count = 1;
         }
         UNLOCK (&local->lock);
 
-        if (stub != NULL) {
-                call_resume (stub);
-        }
-
+        quota_check_limit (frame, loc->parent, this, NULL, NULL);
         return 0;
 
 err:
         QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL);
+                            NULL, NULL);
+
+        return 0;
 
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->symlink,
+                         linkpath, loc, umask, xdata);
         return 0;
 }
 
@@ -1836,10 +2285,9 @@ err:
 int32_t
 quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf)
+                    struct iatt *postbuf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
-        int64_t            delta = 0;
         quota_inode_ctx_t *ctx   = NULL;
 
         if (op_ret < 0) {
@@ -1852,15 +2300,12 @@ quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        delta = (postbuf->ia_blocks - prebuf->ia_blocks) * 512;
-
-        quota_update_size (this, local->loc.inode, NULL, NULL, delta);
-
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (local->loc.inode->gfid));
                 goto out;
         }
@@ -1873,17 +2318,23 @@ quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf);
+                            postbuf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         int32_t          ret   = -1;
         quota_local_t   *local = NULL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto err;
@@ -1898,23 +2349,27 @@ quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         }
 
         STACK_WIND (frame, quota_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
         return 0;
+
 err:
-        QUOTA_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        return 0;
 }
 
 
 int32_t
 quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
-        int64_t            delta = 0;
         quota_inode_ctx_t *ctx   = NULL;
 
         if (op_ret < 0) {
@@ -1927,15 +2382,12 @@ quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        delta = (postbuf->ia_blocks - prebuf->ia_blocks) * 512;
-
-        quota_update_size (this, local->loc.inode, NULL, NULL, delta);
-
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (local->loc.inode->gfid));
                 goto out;
         }
@@ -1948,16 +2400,22 @@ quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf,
-                            postbuf);
+                            postbuf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         quota_local_t   *local = NULL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL)
                 goto err;
@@ -1966,14 +2424,21 @@ quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
 
         local->loc.inode = inode_ref (fd->inode);
 
-        STACK_WIND (frame, quota_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+        STACK_WIND (frame, quota_ftruncate_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate, fd,
+                    offset, xdata);
 
         return 0;
 err:
-        QUOTA_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->ftruncate, fd,
+                         offset, xdata);
+        return 0;
 }
 
 
@@ -1986,14 +2451,23 @@ quota_send_dir_limit_to_cli (call_frame_t *frame, xlator_t *this,
         dict_t            *dict              = NULL;
         quota_inode_ctx_t *ctx               = NULL;
         uint64_t           value             = 0;
+        quota_priv_t      *priv              = NULL;
+
+        priv = this->private;
+        if (!priv->is_quota_on) {
+                snprintf (dir_limit, 1024, "Quota is disabled please turn on");
+                goto dict_set;
+        }
 
         ret = inode_ctx_get (inode, this, &value);
         if (ret < 0)
                 goto out;
 
         ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        snprintf (dir_limit, 1024, "%"PRId64",%"PRId64, ctx->size, ctx->limit);
+        snprintf (dir_limit, 1024, "%"PRId64",%"PRId64, ctx->size,
+                  ctx->hard_lim);
 
+dict_set:
         dict = dict_new ();
         if (dict == NULL) {
                 ret = -1;
@@ -2004,9 +2478,9 @@ quota_send_dir_limit_to_cli (call_frame_t *frame, xlator_t *this,
         if (ret < 0)
                 goto out;
 
-        gf_log (this->name, GF_LOG_INFO, "str = %s", dir_limit);
+        gf_log (this->name, GF_LOG_DEBUG, "str = %s", dir_limit);
 
-        QUOTA_STACK_UNWIND (getxattr, frame, 0, 0, dict);
+        QUOTA_STACK_UNWIND (getxattr, frame, 0, 0, dict, NULL);
 
         ret = 0;
 
@@ -2017,7 +2491,7 @@ out:
 
 int32_t
 quota_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
         int32_t ret     = 0;
 
@@ -2030,14 +2504,14 @@ quota_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         }
 
         STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name);
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
         return 0;
 }
 
 
 int32_t
 quota_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name)
+                const char *name, dict_t *xdata)
 {
         int32_t ret     = 0;
 
@@ -2049,14 +2523,15 @@ quota_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name);
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
         return 0;
 }
 
 
 int32_t
 quota_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2071,12 +2546,17 @@ quota_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "quota context not set in inode (gfid:%s)",
-                        uuid_utoa (local->loc.inode->gfid));
+                if (!IA_ISDIR (buf->ia_type)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "quota context is NULL on "
+                                "inode (%s). "
+                                "If quota is not enabled recently and crawler "
+                                "has finished crawling, its an error",
+                                uuid_utoa (local->loc.inode->gfid));
+                }
+
                 goto out;
         }
 
@@ -2088,17 +2568,22 @@ quota_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
+        QUOTA_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         quota_local_t *local = NULL;
         int32_t        ret   = -1;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto unwind;
@@ -2112,18 +2597,26 @@ quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         STACK_WIND (frame, quota_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
+                    FIRST_CHILD(this)->fops->stat, loc,
+                    xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (stat, frame, -1, ENOMEM, NULL);
+        QUOTA_STACK_UNWIND (stat, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->stat, loc,
+                         xdata);
         return 0;
 }
 
 
 int32_t
 quota_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2138,12 +2631,17 @@ quota_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
-                        uuid_utoa (local->loc.inode->gfid));
+                if (!IA_ISDIR (buf->ia_type)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "quota context is NULL on "
+                                "inode (%s). "
+                                "If quota is not enabled recently and crawler "
+                                "has finished crawling, its an error",
+                                uuid_utoa (local->loc.inode->gfid));
+                }
+
                 goto out;
         }
 
@@ -2155,16 +2653,21 @@ quota_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
+        QUOTA_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int32_t
-quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         quota_local_t *local = NULL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto unwind;
@@ -2174,12 +2677,19 @@ quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
         local->loc.inode = inode_ref (fd->inode);
 
-        STACK_WIND (frame, quota_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
+        STACK_WIND (frame, quota_fstat_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat, fd,
+                    xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fstat, frame, -1, ENOMEM, NULL);
+        QUOTA_STACK_UNWIND (fstat, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fstat, fd,
+                         xdata);
         return 0;
 }
 
@@ -2187,7 +2697,7 @@ unwind:
 int32_t
 quota_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, const char *path,
-                    struct iatt *buf)
+                    struct iatt *buf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2202,11 +2712,12 @@ quota_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (local->loc.inode->gfid));
                 goto out;
         }
@@ -2218,17 +2729,24 @@ quota_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, buf);
+        QUOTA_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, buf,
+                            xdata);
         return 0;
 }
 
 
 int32_t
-quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         quota_local_t *local = NULL;
         int32_t        ret   = -1;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto unwind;
@@ -2242,12 +2760,19 @@ quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
                 goto unwind;
         }
 
-        STACK_WIND (frame, quota_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc, size);
+        STACK_WIND (frame, quota_readlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink, loc,
+                    size, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (readlink, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->readlink, loc,
+                         size, xdata);
         return 0;
 }
 
@@ -2255,7 +2780,8 @@ unwind:
 int32_t
 quota_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref)
+                 int32_t count, struct iatt *buf, struct iobref *iobref,
+                 dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2270,11 +2796,12 @@ quota_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (local->loc.inode->gfid));
                 goto out;
         }
@@ -2287,17 +2814,22 @@ quota_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                            buf, iobref);
+                            buf, iobref, xdata);
         return 0;
 }
 
 
 int32_t
 quota_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, uint32_t flags)
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         quota_local_t *local = NULL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto unwind;
@@ -2307,12 +2839,20 @@ quota_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
         local->loc.inode = inode_ref (fd->inode);
 
-        STACK_WIND (frame, quota_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags);
+        STACK_WIND (frame, quota_readv_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv, fd,
+                    size, offset, flags, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL);
+        QUOTA_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL,
+                            NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->readv, fd,
+                         size, offset, flags, xdata);
         return 0;
 }
 
@@ -2320,7 +2860,7 @@ unwind:
 int32_t
 quota_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
+                 struct iatt *postbuf, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2335,11 +2875,12 @@ quota_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
+                        "inode (%s). "
+                        "If quota is not enabled recently and crawler has "
+                        "finished crawling, its an error",
                         uuid_utoa (local->loc.inode->gfid));
                 goto out;
         }
@@ -2351,16 +2892,23 @@ quota_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        QUOTA_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
 
 int32_t
-quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         quota_local_t *local = NULL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto unwind;
@@ -2371,20 +2919,26 @@ quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
         frame->local = local;
 
         STACK_WIND (frame, quota_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
+                    FIRST_CHILD(this)->fops->fsync, fd,
+                    flags, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fsync, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
 
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fsync, fd,
+                         flags, xdata);
+        return 0;
 }
 
 
 int32_t
 quota_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                   struct iatt *statpost)
+                   struct iatt *statpost, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2399,12 +2953,16 @@ quota_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "quota context not set in inode (gfid:%s)",
-                        uuid_utoa (local->loc.inode->gfid));
+                if (!IA_ISDIR (statpost->ia_type)) {
+                        gf_log (this->name, GF_LOG_DEBUG, "quota context is "
+                                "NULL on inode (%s). "
+                                "If quota is not enabled recently and crawler "
+                                "has finished crawling, its an error",
+                                uuid_utoa (local->loc.inode->gfid));
+                }
+
                 goto out;
         }
 
@@ -2417,18 +2975,23 @@ quota_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
-                            statpost);
+                            statpost, xdata);
         return 0;
 }
 
 
 int32_t
 quota_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         quota_local_t *local = NULL;
         int32_t        ret   = -1;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto unwind;
@@ -2443,11 +3006,18 @@ quota_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         STACK_WIND (frame, quota_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
+                    FIRST_CHILD (this)->fops->setattr, loc,
+                    stbuf, valid, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
+                         FIRST_CHILD (this)->fops->setattr, loc,
+                         stbuf, valid, xdata);
         return 0;
 }
 
@@ -2455,7 +3025,7 @@ unwind:
 int32_t
 quota_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                    struct iatt *statpost)
+                    struct iatt *statpost, dict_t *xdata)
 {
         quota_local_t     *local = NULL;
         quota_inode_ctx_t *ctx   = NULL;
@@ -2470,12 +3040,16 @@ quota_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
-                             &ctx, 0);
+        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
         if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
-                        uuid_utoa (local->loc.inode->gfid));
+                if (!IA_ISDIR (statpost->ia_type)) {
+                        gf_log (this->name, GF_LOG_DEBUG, "quota context is "
+                                "NULL on inode (%s). "
+                                "If quota is not enabled recently and crawler "
+                                "has finished crawling, its an error",
+                                uuid_utoa (local->loc.inode->gfid));
+                }
+
                 goto out;
         }
 
@@ -2487,17 +3061,22 @@ quota_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         QUOTA_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, statpre,
-                            statpost);
+                            statpost, xdata);
         return 0;
 }
 
 
 int32_t
 quota_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         quota_local_t *local = NULL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         local = quota_local_new ();
         if (local == NULL) {
                 goto unwind;
@@ -2507,12 +3086,19 @@ quota_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         local->loc.inode = inode_ref (fd->inode);
 
-        STACK_WIND (frame, quota_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
+        STACK_WIND (frame, quota_fsetattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr, fd,
+                    stbuf, valid, xdata);
         return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL);
+        QUOTA_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
+                         FIRST_CHILD (this)->fops->fsetattr, fd,
+                         stbuf, valid, xdata);
         return 0;
 }
 
@@ -2521,7 +3107,7 @@ int32_t
 quota_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         int32_t            ret    = -1;
         quota_local_t     *local  = NULL;
@@ -2533,7 +3119,7 @@ quota_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto unwind;
         }
 
-        ret = quota_inode_ctx_get (inode, -1, this, NULL, buf, &ctx, 1);
+        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
         if ((ret == -1) || (ctx == NULL)) {
                 gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
                         "context in inode (gfid:%s)", uuid_utoa (inode->gfid));
@@ -2563,17 +3149,18 @@ unlock:
 
 unwind:
         QUOTA_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
-                            buf, preparent, postparent);
+                            buf, preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 quota_mknod_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    mode_t mode, dev_t rdev, dict_t *parms)
+                    mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
         quota_local_t *local    = NULL;
         int32_t        op_errno = EINVAL;
+        quota_priv_t  *priv     = NULL;
 
         local = frame->local;
         if (local == NULL) {
@@ -2581,30 +3168,38 @@ quota_mknod_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto unwind;
         }
 
+        priv = this->private;
+
         if (local->op_ret == -1) {
                 op_errno = local->op_errno;
                 goto unwind;
         }
 
-        STACK_WIND (frame, quota_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, parms);
+        STACK_WIND (frame, quota_mknod_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod, loc,
+                    mode, rdev, umask, xdata);
 
         return 0;
 
 unwind:
         QUOTA_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL);
+                            NULL, NULL, NULL);
         return 0;
 }
 
 
 int
 quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dev_t rdev, dict_t *parms)
+             dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int32_t            ret            = -1;
-        quota_local_t     *local          = NULL;
-        call_stub_t       *stub           = NULL;
+        quota_priv_t  *priv     = NULL;
+        int32_t        ret      = -1;
+        quota_local_t *local    = NULL;
+        call_stub_t   *stub     = NULL;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
         local = quota_local_new ();
         if (local == NULL) {
@@ -2620,185 +3215,317 @@ quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         }
 
         stub = fop_mknod_stub (frame, quota_mknod_helper, loc, mode, rdev,
-                               parms);
+                               umask, xdata);
         if (stub == NULL) {
                 goto err;
         }
 
-        local->link_count = 1;
-        local->stub = stub;
-        local->delta = 0;
-
-        quota_check_limit (frame, loc->parent, this, NULL, NULL);
-
-        stub = NULL;
-
         LOCK (&local->lock);
         {
-                local->link_count = 0;
-                if (local->validate_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
+                local->link_count = 1;
+                local->stub = stub;
+                local->delta = 0;
         }
         UNLOCK (&local->lock);
 
-        if (stub != NULL) {
-                call_resume (stub);
-        }
-
+        quota_check_limit (frame, loc->parent, this, NULL, NULL);
         return 0;
+
 err:
-        QUOTA_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+        QUOTA_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
 
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->mknod, loc,
+                         mode, rdev, umask, xdata);
         return 0;
 }
 
 int
 quota_setxattr_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int op_ret, int op_errno)
+                    xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+        int                ret   = 0;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        ret = quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
+        if ((ret < 0) || (ctx == NULL)) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->hard_lim = local->limit.hard_lim;
+                ctx->soft_lim = local->limit.soft_lim_percent;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 quota_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int flags)
+                loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
-        int             op_errno = EINVAL;
-        int             op_ret   = -1;
+        quota_priv_t  *priv     = NULL;
+        int            op_errno = EINVAL;
+        int            op_ret   = -1;
+        int64_t        hard_lim = -1, soft_lim = -1;
+        quota_local_t *local    = NULL;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
-                                   trav, op_errno, err);
+        if (frame->root->pid >= 0) {
+                GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
+                                           op_errno, err);
+                GF_IF_INTERNAL_XATTR_GOTO ("trusted.pgfid*", dict, op_errno,
+                                           err);
+        }
+
+        quota_get_limits (this, dict, &hard_lim, &soft_lim);
+
+        if (hard_lim > 0) {
+                local = quota_local_new ();
+                if (local == NULL) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+
+                frame->local = local;
+                loc_copy (&local->loc, loc);
+
+                local->limit.hard_lim = hard_lim;
+                local->limit.soft_lim_percent = soft_lim;
+        }
 
         STACK_WIND (frame, quota_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr, loc,
+                    dict, flags, xdata);
         return 0;
 err:
-        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->setxattr, loc,
+                         dict, flags, xdata);
         return 0;
 }
 
 int
 quota_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno)
+                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+        quota_inode_ctx_t *ctx   = NULL;
+        quota_local_t     *local = NULL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_ret = quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
+        if ((op_ret < 0) || (ctx == NULL)) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->hard_lim = local->limit.hard_lim;
+                ctx->soft_lim = local->limit.soft_lim_percent;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 quota_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int flags)
+                 dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
-        int32_t         op_ret   = -1;
-        int32_t         op_errno = EINVAL;
+        quota_priv_t  *priv     = NULL;
+        int32_t        op_ret   = -1;
+        int32_t        op_errno = EINVAL;
+        quota_local_t *local    = NULL;
+        int64_t        hard_lim = -1, soft_lim = -1;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
-                                   trav, op_errno, err);
+        if (0 <= frame->root->pid) {
+                GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*",
+                                            dict, op_errno, err);
+                GF_IF_INTERNAL_XATTR_GOTO ("trusted.pgfid*", dict,
+                                            op_errno, err);
+        }
+
+        quota_get_limits (this, dict, &hard_lim, &soft_lim);
+
+        if (hard_lim > 0) {
+                local = quota_local_new ();
+                if (local == NULL) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                frame->local = local;
+                local->loc.inode = inode_ref (fd->inode);
+
+                local->limit.hard_lim = hard_lim;
+                local->limit.soft_lim_percent = soft_lim;
+        }
 
         STACK_WIND (frame, quota_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr, fd,
+                    dict, flags, xdata);
         return 0;
- err:
-        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+err:
+        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fsetxattr, fd,
+                         dict, flags, xdata);
         return 0;
 }
 
 
 int
-quota_removexattr_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno)
+quota_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (removexattr, frame, op_ret, op_errno);
+        QUOTA_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 quota_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name)
+                   loc_t *loc, const char *name, dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         int32_t         op_errno = EINVAL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         VALIDATE_OR_GOTO (this, err);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.quota*",
-                                 name, op_errno, err);
+        /* all quota xattrs can be cleaned up by doing setxattr on special key.
+         * Hence its ok that we don't allow removexattr on quota keys here.
+         */
+        if (frame->root->pid >= 0) {
+                GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.quota*",
+                                         name, op_errno, err);
+                GF_IF_NATIVE_XATTR_GOTO ("trusted.pgfid*", name,
+                                         op_errno, err);
+        }
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (loc, err);
 
         STACK_WIND (frame, quota_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name);
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
         return 0;
+
 err:
-        QUOTA_STACK_UNWIND (removexattr, frame, -1,  op_errno);
+        QUOTA_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->removexattr,
+                         loc, name, xdata);
         return 0;
 }
 
 
 int
-quota_fremovexattr_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret, int32_t op_errno)
+quota_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno);
+        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 int
 quota_fremovexattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, const char *name)
+                    fd_t *fd, const char *name, dict_t *xdata)
 {
+        quota_priv_t       *priv        = NULL;
         int32_t         op_ret   = -1;
         int32_t         op_errno = EINVAL;
 
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.quota*",
-                                 name, op_errno, err);
-
+        if (frame->root->pid >= 0) {
+                GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.quota*",
+                                         name, op_errno, err);
+                GF_IF_NATIVE_XATTR_GOTO ("trusted.pgfid*", name,
+                                         op_errno, err);
+        }
         STACK_WIND (frame, quota_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name);
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
         return 0;
- err:
-        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno);
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fremovexattr,
+                         fd, name, xdata);
         return 0;
 }
 
+
 int32_t
 quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                  dict_t *xdata)
 {
-	inode_t            *root_inode = NULL;
-        quota_priv_t       *priv       = NULL;
-        uint64_t            value      = 0;
-        quota_inode_ctx_t  *ctx        = NULL;
-        limits_t           *limit_node = NULL;
-	int64_t             usage      = -1;
-	int64_t             avail      = -1;
+	inode_t           *inode  = NULL;
+        uint64_t           value  = 0;
+	int64_t            usage  = -1;
+	int64_t            avail  = -1;
+        int64_t            blocks = 0;
+        quota_inode_ctx_t *ctx    = NULL;
+        int                ret    = 0;
+        gf_boolean_t       dict_created = _gf_false;
 
-	root_inode = cookie;
+        inode = cookie;
 
         /* This fop will fail mostly in case of client disconnect's,
          * which is already logged. Hence, not logging here */
@@ -2809,124 +3536,540 @@ quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	 * cookie, and it would only do so if the value was non-NULL.  This
 	 * check is therefore just routine defensive coding.
 	 */
-	if (!root_inode) {
+	if (!inode) {
 		gf_log(this->name,GF_LOG_WARNING,
 		       "null inode, cannot adjust for quota");
 		goto unwind;
 	}
-	if (!root_inode->table || (root_inode != root_inode->table->root)) {
-		gf_log(this->name,GF_LOG_WARNING,
-		       "non-root inode, cannot adjust for quota");
-		goto unwind;
-	}
 
-        inode_ctx_get (root_inode, this, &value);
+        inode_ctx_get (inode, this, &value);
 	if (!value) {
 		goto unwind;
 	}
+
+        /* if limit is set on this inode, report statfs based on this inode
+         * else report based on root.
+         */
         ctx = (quota_inode_ctx_t *)(unsigned long)value;
-	usage = (ctx->size) / buf->f_bsize;
-        priv = this->private;
+        if (ctx->hard_lim <= 0) {
+                inode_ctx_get (inode->table->root, this, &value);
+                ctx = (quota_inode_ctx_t *)(unsigned long) value;
+                if (!ctx || ctx->hard_lim < 0)
+                        goto unwind;
+        }
+
+        { /* statfs is adjusted in this code block */
+                usage = (ctx->size) / buf->f_bsize;
+
+                blocks = ctx->hard_lim / buf->f_bsize;
+                buf->f_blocks = blocks;
+
+                avail = buf->f_blocks - usage;
+                avail = max (avail, 0);
+
+                buf->f_bfree = avail;
+                /*
+                 * We have to assume that the total assigned quota
+                 * won't cause us to dip into the reserved space,
+                 * because dealing with the overcommitted cases is
+                 * just too hairy (especially when different bricks
+                 * might be using different reserved percentages and
+                 * such).
+                 */
+                buf->f_bavail = buf->f_bfree;
+        }
 
-        list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
-		/* Notice that this only works for volume-level quota. */
-                if (strcmp (limit_node->path, "/") == 0) {
-			buf->f_blocks = limit_node->value / buf->f_bsize;
-			avail = buf->f_blocks - usage;
-			if (buf->f_bfree > avail) {
-				buf->f_bfree = avail;
-			}
-			/*
-			 * We have to assume that the total assigned quota
-			 * won't cause us to dip into the reserved space,
-			 * because dealing with the overcommitted cases is
-			 * just too hairy (especially when different bricks
-			 * might be using different reserved percentages and
-			 * such).
-			 */
-			buf->f_bavail = buf->f_bfree;
-			break;
-                }
+        if (!xdata) {
+                xdata = dict_new ();
+                if (!xdata)
+                        goto unwind;
+                dict_created = _gf_true;
         }
 
+        ret = dict_set_int8 (xdata, "quota-deem-statfs", 1);
+        if (-1 == ret)
+                gf_log (this->name, GF_LOG_ERROR, "Dict set failed, "
+                        "deem-statfs option may have no effect");
+
 unwind:
-	if (root_inode) {
-		inode_unref(root_inode);
-	}
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        QUOTA_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
+
+        if (dict_created)
+                dict_unref (xdata);
         return 0;
 }
 
 
 int32_t
-quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+quota_statfs_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
-	inode_t *root_inode = NULL;
+        quota_local_t   *local          = NULL;
+        int              op_errno       = EINVAL;
 
-	if (loc->inode) {
-		root_inode = loc->inode->table->root;
-		inode_ref(root_inode);
-		STACK_WIND_COOKIE (frame, quota_statfs_cbk, root_inode,
-				   FIRST_CHILD(this),
-				   FIRST_CHILD(this)->fops->statfs, loc);
-	}
-	else {
-		/*
-		 * We have to make sure that we never get to quota_statfs_cbk
-		 * with a cookie that points to something other than an inode,
-		 * which is exactly what would happen with STACK_UNWIND using
-		 * that as a callback.  Therefore, use default_statfs_cbk in
-		 * this case instead.
-		 */
-		gf_log(this->name,GF_LOG_WARNING,
-		       "missing inode, cannot adjust for quota");
-		STACK_WIND (frame, default_statfs_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->statfs, loc);
-	}
+        GF_VALIDATE_OR_GOTO ("quota", (local = frame->local), err);
+
+        if (-1 == local->op_ret) {
+                op_errno = local->op_errno;
+                goto err;
+        }
+
+        STACK_WIND_COOKIE (frame, quota_statfs_cbk, loc->inode,
+                           FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+int32_t
+quota_statfs_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
+{
+        quota_local_t     *local      = NULL;
+        int32_t            ret        = 0;
+        quota_inode_ctx_t *ctx        = NULL;
+        int64_t           *size       = 0;
+        uint64_t           value      = 0;
+
+        local = frame->local;
+
+        if (op_ret < 0)
+                goto resume;
+
+        GF_ASSERT (local);
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, resume, op_errno,
+                                        EINVAL);
+        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, resume, op_errno,
+                                        EINVAL);
+
+        ret = inode_ctx_get (local->validate_loc.inode, this, &value);
+
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context is not present in inode (gfid:%s)",
+                        uuid_utoa (local->validate_loc.inode->gfid));
+                op_errno = EINVAL;
+                goto resume;
+        }
+
+        ret = dict_get_bin (xdata, QUOTA_SIZE_KEY, (void **) &size);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "size key not present in dict");
+                op_errno = EINVAL;
+                goto resume;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->size = ntoh64 (*size);
+                gettimeofday (&ctx->tv, NULL);
+        }
+        UNLOCK (&ctx->lock);
+
+resume:
+        quota_link_count_decrement (local);
         return 0;
 }
 
+int32_t
+quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        quota_local_t   *local  = NULL;
+        int              op_errno       = 0;
+        call_stub_t     *stub           = NULL;
+        quota_priv_t *priv  = NULL;
+        int           ret       = 0;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
+	if (priv->consider_statfs && loc->inode) {
+                local = quota_local_new ();
+                if (!local) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                frame->local = local;
+
+                stub = fop_statfs_stub (frame, quota_statfs_helper, loc, xdata);
+                if (!stub) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+
+                LOCK (&local->lock);
+                {
+                        local->inode = inode_ref (loc->inode);
+                        local->link_count = 1;
+                        local->stub = stub;
+                }
+                UNLOCK (&local->lock);
+
+                ret = quota_validate (frame, local->inode, this,
+                                      quota_statfs_validate_cbk);
+                if (0 > ret) {
+                        quota_handle_validate_error (local, -1, -ret);
+                }
+
+                return 0;
+	}
+
+        /*
+         * We have to make sure that we never get to quota_statfs_cbk
+         * with a cookie that points to something other than an inode,
+         * which is exactly what would happen with STACK_UNWIND using
+         * that as a callback.  Therefore, use default_statfs_cbk in
+         * this case instead.
+         *
+         * Also if the option deem-statfs is not set to "on" don't
+         * bother calculating quota limit on / in statfs_cbk.
+         */
+        if (priv->consider_statfs)
+                gf_log (this->name,GF_LOG_WARNING,
+                        "missing inode, cannot adjust for quota");
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL, NULL);
+
+        if (local)
+                quota_local_cleanup (this, local);
+        return 0;
+}
 
 int
 quota_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, gf_dirent_t *entries)
+                    int op_ret, int op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
-        gf_dirent_t *entry = NULL;
+        gf_dirent_t   *entry = NULL;
+        quota_local_t *local = NULL;
+        loc_t          loc   = {0, };
 
         if (op_ret <= 0)
                 goto unwind;
 
+        local = frame->local;
+
         list_for_each_entry (entry, &entries->list, list) {
-                /* TODO: fill things */
+                if ((strcmp (entry->d_name, ".") == 0)
+                    || (strcmp (entry->d_name, "..") == 0))
+                        continue;
+
+                uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
+                loc.inode = inode_ref (entry->inode);
+                loc.parent = inode_ref (local->loc.inode);
+                uuid_copy (loc.pargfid, loc.parent->gfid);
+                loc.name = entry->d_name;
+
+                quota_fill_inodectx (this, entry->inode, entry->dict,
+                                     &loc, &entry->d_stat, &op_errno);
+
+                loc_wipe (&loc);
         }
 
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        QUOTA_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, xdata);
 
         return 0;
 }
+
 int
 quota_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 off_t offset, dict_t *dict)
 {
-        int ret = 0;
+        quota_priv_t  *priv     = NULL;
+        int            ret      = 0;
+        gf_boolean_t   new_dict = _gf_false;
+        quota_local_t *local    = NULL;
+
+        priv = this->private;
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
+        local = quota_local_new ();
+
+        if (local == NULL) {
+                goto err;
+        }
+
+        frame->local = local;
+
+        local->loc.inode = inode_ref (fd->inode);
+
+        if (dict == NULL) {
+                dict = dict_new ();
+                new_dict = _gf_true;
+        }
 
         if (dict) {
-                ret = dict_set_uint64 (dict, QUOTA_SIZE_KEY, 0);
+                ret = dict_set_int8 (dict, QUOTA_LIMIT_KEY, 1);
                 if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "dict set of key for hard-limit failed");
                         goto err;
                 }
         }
 
         STACK_WIND (frame, quota_readdirp_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp, fd,
+                    size, offset, dict);
+
+        if (new_dict) {
+                dict_unref (dict);
+        }
+
         return 0;
 err:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, EINVAL, NULL);
+        STACK_UNWIND_STRICT (readdirp, frame, -1, EINVAL, NULL, NULL);
+
+        if (new_dict) {
+                dict_unref (dict);
+        }
+
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->readdirp, fd,
+                         size, offset, dict);
         return 0;
 }
 
+int32_t
+quota_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t                  ret            = 0;
+        uint64_t                 ctx_int        = 0;
+        quota_inode_ctx_t       *ctx            = NULL;
+        quota_local_t           *local          = NULL;
+
+        local = frame->local;
+
+        if ((op_ret < 0) || (local == NULL)) {
+                goto out;
+        }
+
+        ret = inode_ctx_get (local->loc.inode, this, &ctx_int);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get the context", local->loc.path);
+                goto out;
+        }
+
+        ctx = (quota_inode_ctx_t *)(unsigned long) ctx_int;
+
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in %s (gfid:%s)",
+                        local->loc.path, uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *postbuf;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+
+        return 0;
+}
+
+
+int32_t
+quota_fallocate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        int32_t mode, off_t offset, size_t len, dict_t *xdata)
+{
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
+        quota_priv_t  *priv     = NULL;
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
+
+        priv = this->private;
+
+        if (local->op_ret == -1) {
+                op_errno = local->op_errno;
+                goto unwind;
+        }
+
+        STACK_WIND (frame, quota_fallocate_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		    xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		off_t offset, size_t len, dict_t *xdata)
+{
+        int32_t            ret     = -1, op_errno = EINVAL;
+        int32_t            parents = 0;
+        quota_local_t     *local   = NULL;
+        quota_inode_ctx_t *ctx     = NULL;
+        quota_priv_t      *priv    = NULL;
+        quota_dentry_t    *dentry  = NULL;
+        call_stub_t       *stub    = NULL;
+
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
+
+        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO ("quota", this, unwind);
+        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+        local->loc.inode = inode_ref (fd->inode);
+
+        ret = quota_inode_ctx_get (fd->inode, this, &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_DEBUG, "quota context is "
+                        "NULL on inode (%s). "
+                        "If quota is not enabled recently and crawler "
+                        "has finished crawling, its an error",
+                        uuid_utoa (local->loc.inode->gfid));
+        }
+
+        stub = fop_fallocate_stub(frame, quota_fallocate_helper, fd, mode,
+                                  offset, len, xdata);
+        if (stub == NULL) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
+
+        if (ctx != NULL) {
+                LOCK (&ctx->lock);
+                {
+                        list_for_each_entry (dentry, &ctx->parents, next) {
+                                parents++;
+                        }
+                }
+                UNLOCK (&ctx->lock);
+        }
+
+	/*
+	 * Note that by using len as the delta we're assuming the range from
+	 * offset to offset+len has not already been allocated. This can result
+	 * in ENOSPC errors attempting to allocate an already allocated range.
+	 */
+        local->delta = len;
+        local->stub = stub;
+        local->link_count = parents;
+
+        if (parents == 0) {
+                local->link_count = 1;
+                quota_check_limit (frame, fd->inode, this, NULL, NULL);
+        } else {
+                list_for_each_entry (dentry, &ctx->parents, next) {
+                        quota_check_limit (frame, fd->inode, this, dentry->name,
+                                           dentry->par);
+                }
+        }
+
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+
+off:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->fallocate, fd, mode, offset,
+                         len, xdata);
+        return 0;
+}
+
+/* Logs if
+*  i.   Usage crossed soft limit
+*  ii.  Usage above soft limit and alert-time elapsed
+*/
+void
+quota_log_usage (xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
+                 int64_t delta)
+{
+        struct timeval           cur_time       = {0,};
+        char                    *usage_str      = NULL;
+        char                    size_str[32]    = {0};
+        char                    *path           = NULL;
+        int64_t                  cur_size       = 0;
+        quota_priv_t            *priv           = NULL;
+        gf_boolean_t            dyn_mem         = _gf_true;
+
+        priv = this->private;
+        if ((ctx->soft_lim <= 0) || (timerisset (&ctx->prev_log) &&
+                                     !quota_timeout (&ctx->prev_log,
+                                                     priv->log_timeout))) {
+                return;
+        }
+
+
+        cur_size = ctx->size + delta;
+        usage_str = gf_uint64_2human_readable (cur_size);
+        if (!usage_str) {
+                snprintf (size_str, sizeof (size_str), "%"PRId64, cur_size);
+                usage_str = (char*) size_str;
+                dyn_mem = _gf_false;
+        }
+        inode_path (inode, NULL, &path);
+        if (!path)
+                path = uuid_utoa (inode->gfid);
+
+        gettimeofday (&cur_time, NULL);
+        /* Usage crossed/reached soft limit */
+        if (DID_REACH_LIMIT (ctx->soft_lim, ctx->size, cur_size)) {
+
+                gf_log (this->name, GF_LOG_ALERT, "Usage crossed "
+                        "soft limit: %s used by %s", usage_str, path);
+                ctx->prev_log = cur_time;
+        }
+        /* Usage is above soft limit */
+        else if (cur_size > ctx->soft_lim){
+                gf_log (this->name, GF_LOG_ALERT, "Usage is above "
+                        "soft limit: %s used by %s", usage_str, path);
+                ctx->prev_log = cur_time;
+        }
+
+        if (dyn_mem)
+                GF_FREE (usage_str);
+}
 
 int32_t
 mem_acct_init (xlator_t *this)
@@ -2979,59 +4122,6 @@ quota_forget (xlator_t *this, inode_t *inode)
         return 0;
 }
 
-
-int
-quota_parse_limits (quota_priv_t *priv, xlator_t *this, dict_t *xl_options)
-{
-        int32_t       ret       = -1;
-        char         *str       = NULL;
-        char         *str_val   = NULL;
-        char         *path      = NULL;
-        uint64_t      value     = 0;
-        limits_t     *quota_lim = NULL;
-
-        ret = dict_get_str (xl_options, "limit-set", &str);
-
-        if (str) {
-                path = strtok (str, ":");
-
-                while (path) {
-                        str_val = strtok (NULL, ",");
-
-                        ret = gf_string2bytesize (str_val, &value);
-                        if (ret != 0)
-                                goto err;
-
-                        QUOTA_ALLOC_OR_GOTO (quota_lim, limits_t, err);
-
-                        quota_lim->path = path;
-
-                        quota_lim->value = value;
-
-                        gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64,
-                                quota_lim->path, quota_lim->value);
-
-                        list_add_tail (&quota_lim->limit_list,
-                                       &priv->limit_head);
-
-                        path = strtok (NULL, ":");
-                }
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no \"limit-set\" option provided");
-        }
-
-        list_for_each_entry (quota_lim, &priv->limit_head, limit_list) {
-                gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64, quota_lim->path,
-                        quota_lim->value);
-        }
-
-        ret = 0;
-err:
-        return ret;
-}
-
-
 int32_t
 init (xlator_t *this)
 {
@@ -3053,17 +4143,18 @@ init (xlator_t *this)
 
         QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err);
 
-        INIT_LIST_HEAD (&priv->limit_head);
+        LOCK_INIT (&priv->lock);
 
         this->private = priv;
 
-        ret = quota_parse_limits (priv, this, this->options);
-
-        if (ret) {
-                goto err;
-        }
-
-        GF_OPTION_INIT ("timeout", priv->timeout, int64, err);
+        GF_OPTION_INIT ("deem-statfs", priv->consider_statfs, bool, err);
+        GF_OPTION_INIT ("server-quota", priv->is_quota_on, bool, err);
+        GF_OPTION_INIT ("default-soft-limit", priv->default_soft_lim, percent,
+                        err);
+        GF_OPTION_INIT ("soft-timeout", priv->soft_timeout, time, err);
+        GF_OPTION_INIT ("hard-timeout", priv->hard_timeout, time, err);
+        GF_OPTION_INIT ("alert-time", priv->log_timeout, time, err);
+        GF_OPTION_INIT ("volume-uuid", priv->volume_uuid, str, err);
 
         this->local_pool = mem_pool_new (quota_local_t, 64);
         if (!this->local_pool) {
@@ -3073,43 +4164,102 @@ init (xlator_t *this)
                 goto err;
         }
 
+        if (priv->is_quota_on) {
+                priv->rpc_clnt = quota_enforcer_init (this, this->options);
+                if (priv->rpc_clnt == NULL) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "quota enforcer rpc init failed");
+                        goto err;
+                }
+        }
+
         ret = 0;
 err:
         return ret;
 }
 
-
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
-        int32_t          ret    = -1;
-        quota_priv_t    *priv   = NULL;
-        limits_t        *limit  = NULL;
-        limits_t        *next   = NULL;
+        int32_t       ret      = -1;
+        quota_priv_t *priv     = NULL;
+        gf_boolean_t  quota_on = _gf_false;
 
         priv = this->private;
 
-        list_for_each_entry_safe (limit, next, &priv->limit_head, limit_list) {
-                list_del (&limit->limit_list);
-
-                GF_FREE (limit);
-        }
+        GF_OPTION_RECONF ("deem-statfs", priv->consider_statfs, options, bool,
+                          out);
+        GF_OPTION_RECONF ("server-quota", quota_on, options, bool,
+                          out);
+        GF_OPTION_RECONF ("default-soft-limit", priv->default_soft_lim,
+                          options, percent, out);
+        GF_OPTION_RECONF ("alert-time", priv->log_timeout, options,
+                          time, out);
+        GF_OPTION_RECONF ("soft-timeout", priv->soft_timeout, options,
+                          time, out);
+        GF_OPTION_RECONF ("hard-timeout", priv->hard_timeout, options,
+                          time, out);
+
+        if (quota_on) {
+                priv->rpc_clnt = quota_enforcer_init (this,
+                                                      this->options);
+                if (priv->rpc_clnt == NULL) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "quota enforcer rpc init failed");
+                        goto out;
+                }
 
-        ret = quota_parse_limits (priv, this, options);
-        if (ret == -1) {
-                gf_log ("quota", GF_LOG_WARNING,
-                        "quota reconfigure failed, "
-                        "new changes will not take effect");
-                goto out;
+        } else {
+                if (priv->rpc_clnt) {
+                        // Quotad is shutdown when there is no started volume
+                        // which has quota enabled. So, we should disable the
+                        // enforcer client when quota is disabled on a volume,
+                        // to avoid spurious reconnect attempts to a service
+                        // (quotad), that is known to be down.
+                        rpc_clnt_disable (priv->rpc_clnt);
+                }
         }
 
-        GF_OPTION_RECONF ("timeout", priv->timeout, options, int64, out);
+        priv->is_quota_on = quota_on;
 
         ret = 0;
 out:
         return ret;
 }
 
+int32_t
+quota_priv_dump (xlator_t *this)
+{
+        quota_priv_t *priv = NULL;
+        int32_t       ret  = -1;
+
+
+        GF_ASSERT (this);
+
+        priv = this->private;
+
+        gf_proc_dump_add_section ("xlators.features.quota.priv", this->name);
+
+        ret = TRY_LOCK (&priv->lock);
+        if (ret)
+             goto out;
+        else {
+                gf_proc_dump_write("soft-timeout", "%d", priv->soft_timeout);
+                gf_proc_dump_write("hard-timeout", "%d", priv->hard_timeout);
+                gf_proc_dump_write("alert-time", "%d", priv->log_timeout);
+                gf_proc_dump_write("quota-on", "%d", priv->is_quota_on);
+                gf_proc_dump_write("statfs", "%d", priv->consider_statfs);
+                gf_proc_dump_write("volume-uuid", "%s", priv->volume_uuid);
+                gf_proc_dump_write("validation-count", "%ld",
+                                    priv->validation_count);
+        }
+        UNLOCK (&priv->lock);
+
+out:
+        return 0;
+}
 
 void
 fini (xlator_t *this)
@@ -3145,19 +4295,79 @@ struct xlator_fops fops = {
         .removexattr  = quota_removexattr,
         .fremovexattr = quota_fremovexattr,
         .readdirp     = quota_readdirp,
+	.fallocate    = quota_fallocate,
 };
 
 struct xlator_cbks cbks = {
         .forget = quota_forget
 };
 
+struct xlator_dumpops dumpops = {
+        .priv    = quota_priv_dump,
+};
 struct volume_options options[] = {
         {.key = {"limit-set"}},
-        {.key = {"timeout"},
-         .type = GF_OPTION_TYPE_SIZET,
-         .default_value = "0",
-         .description = "quota caches the directory sizes on client. Timeout "
-                        "indicates the timeout for the cache to be revalidated."
+        {.key = {"deem-statfs"},
+         .type = GF_OPTION_TYPE_BOOL,
+         .default_value = "off",
+         .description = "If set to on, it takes quota limits into"
+                        "consideration while estimating fs size. (df command)"
+                        " (Default is off)."
+        },
+        {.key = {"server-quota"},
+         .type = GF_OPTION_TYPE_BOOL,
+         .default_value = "off",
+         .description = "Skip the quota enforcement if the feature is"
+                        " not turned on. This is not a user exposed option."
+        },
+        {.key = {"default-soft-limit"},
+         .type = GF_OPTION_TYPE_PERCENT,
+         .default_value = "80%",
+        },
+        {.key = {"soft-timeout"},
+         .type = GF_OPTION_TYPE_TIME,
+         .min = 0,
+         .max = 1800,
+         .default_value = "60",
+         .description = "quota caches the directory sizes on client. "
+                        "soft-timeout indicates the timeout for the validity of"
+                        " cache before soft-limit has been crossed."
+        },
+        {.key = {"hard-timeout"},
+         .type = GF_OPTION_TYPE_TIME,
+         .min = 0,
+         .max = 60,
+         .default_value = "5",
+         .description = "quota caches the directory sizes on client. "
+                        "hard-timeout indicates the timeout for the validity of"
+                        " cache after soft-limit has been crossed."
+        },
+        { .key   = {"username"},
+          .type  = GF_OPTION_TYPE_ANY,
+        },
+        { .key   = {"password"},
+          .type  = GF_OPTION_TYPE_ANY,
+        },
+        { .key   = {"transport-type"},
+          .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp",
+                    "tcp/client", "ib-verbs/client", "rdma"},
+          .type  = GF_OPTION_TYPE_STR,
+        },
+        { .key   = {"remote-host"},
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS,
+        },
+        { .key   = {"remote-port"},
+          .type  = GF_OPTION_TYPE_INT,
+        },
+        { .key  = {"volume-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "uuid of the volume this brick is part of."
+        },
+        { .key  = {"alert-time"},
+          .type = GF_OPTION_TYPE_TIME,
+          .min = 0,
+          .max = 7*86400,
+          .default_value = "86400",
         },
         {.key = {NULL}}
 };
diff --git a/xlators/features/quota/src/quota.h b/xlators/features/quota/src/quota.h
index 79b4d07e3..84c3257fe 100644
--- a/xlators/features/quota/src/quota.h
+++ b/xlators/features/quota/src/quota.h
@@ -1,50 +1,58 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
+#ifndef _QUOTA_H
+#define _QUOTA_H
+
 #include "xlator.h"
 #include "call-stub.h"
 #include "defaults.h"
-#include "byte-order.h"
 #include "common-utils.h"
 #include "quota-mem-types.h"
+#include "glusterfs.h"
+#include "compat.h"
+#include "logging.h"
+#include "dict.h"
+#include "stack.h"
+#include "common-utils.h"
+#include "event.h"
+#include "globals.h"
+#include "rpcsvc.h"
+#include "rpc-clnt.h"
+#include "byte-order.h"
+#include "glusterfs3-xdr.h"
+#include "glusterfs3.h"
+#include "xdr-generic.h"
+#include "compat-errno.h"
+#include "protocol-common.h"
 
-#define QUOTA_XATTR_PREFIX      "trusted."
 #define DIRTY                   "dirty"
 #define SIZE                    "size"
 #define CONTRIBUTION            "contri"
 #define VAL_LENGTH              8
 #define READDIR_BUF             4096
 
-#define QUOTA_STACK_DESTROY(_frame, _this)              \
-        do {                                            \
-                quota_local_t *_local = NULL;           \
-                _local = _frame->local;                 \
-                _frame->local = NULL;                   \
-                STACK_DESTROY (_frame->root);           \
-                quota_local_cleanup (_this, _local);    \
-                GF_FREE (_local);                       \
-        } while (0)
+#ifndef UUID_CANONICAL_FORM_LEN
+#define UUID_CANONICAL_FORM_LEN 36
+#endif
+
+#define WIND_IF_QUOTAOFF(is_quota_on, label)     \
+        if (!is_quota_on)                       \
+                goto label;
+
+#define DID_REACH_LIMIT(lim, prev_size, cur_size)               \
+        ((cur_size) >= (lim) && (prev_size) < (lim))
 
 #define QUOTA_SAFE_INCREMENT(lock, var)         \
         do {                                    \
@@ -66,7 +74,7 @@
                                  gf_quota_mt_##type);   \
                 if (!var) {                             \
                         gf_log ("", GF_LOG_ERROR,       \
-                                "out of memory :(");    \
+                                "out of memory");    \
                         ret = -1;                       \
                         goto label;                     \
                 }                                       \
@@ -83,7 +91,6 @@
                 }                                                       \
                 STACK_UNWIND_STRICT (fop, frame, params);               \
                 quota_local_cleanup (_this, _local);                    \
-                mem_put (_local);                                       \
         } while (0)
 
 #define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
@@ -95,11 +102,17 @@
 #define GET_CONTRI_KEY(var, _vol_name, _gfid, _ret)             \
         do {                                                    \
                 char _gfid_unparsed[40];                        \
-                uuid_unparse (_gfid, _gfid_unparsed);           \
-                _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
-                                    "%s.%s." CONTRIBUTION,      \
-                                    _vol_name, _gfid_unparsed); \
-        } while (0)
+                if (_gfid != NULL) {                            \
+                        uuid_unparse (_gfid, _gfid_unparsed);   \
+                        _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
+                                            "%s.%s." CONTRIBUTION,      \
+                                            _vol_name, _gfid_unparsed); \
+                } else {                                                \
+                        _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
+                                            "%s.." CONTRIBUTION,       \
+                                            _vol_name);             \
+                }                                                       \
+         } while (0)
 
 
 #define GET_CONTRI_KEY_OR_GOTO(var, _vol_name, _gfid, label)    \
@@ -117,6 +130,11 @@
                         goto label;                             \
         } while (0)
 
+#define QUOTA_REG_OR_LNK_FILE(ia_type)  \
+    (IA_ISREG (ia_type) || IA_ISLNK (ia_type))
+
+
+
 struct quota_dentry {
         char            *name;
         uuid_t           par;
@@ -126,44 +144,77 @@ typedef struct quota_dentry quota_dentry_t;
 
 struct quota_inode_ctx {
         int64_t          size;
-        int64_t          limit;
+        int64_t          hard_lim;
+        int64_t          soft_lim;
         struct iatt      buf;
         struct list_head parents;
         struct timeval   tv;
+        struct timeval   prev_log;
         gf_lock_t        lock;
 };
 typedef struct quota_inode_ctx quota_inode_ctx_t;
 
+struct quota_limit {
+        int64_t hard_lim;
+        int64_t soft_lim_percent;
+} __attribute__ ((packed));
+typedef struct quota_limit quota_limit_t;
+
+typedef void
+(*quota_ancestry_built_t) (struct list_head *parents, inode_t *inode,
+                           int32_t op_ret, int32_t op_errno, void *data);
+
 struct quota_local {
-        gf_lock_t    lock;
-        uint32_t     validate_count;
-        uint32_t     link_count;
-        loc_t        loc;
-        loc_t        oldloc;
-        loc_t        newloc;
-        loc_t        validate_loc;
-        int64_t      delta;
-        int32_t      op_ret;
-        int32_t      op_errno;
-        int64_t      size;
-        int64_t      limit;
-        char         just_validated;
-        inode_t     *inode;
-        call_stub_t *stub;
+        gf_lock_t               lock;
+        uint32_t                validate_count;
+        uint32_t                link_count;
+        loc_t                   loc;
+        loc_t                   oldloc;
+        loc_t                   newloc;
+        loc_t                   validate_loc;
+        int64_t                 delta;
+        int32_t                 op_ret;
+        int32_t                 op_errno;
+        int64_t                 size;
+        gf_boolean_t            skip_check;
+        char                    just_validated;
+        fop_lookup_cbk_t        validate_cbk;
+        inode_t                *inode;
+        call_stub_t            *stub;
+        struct iobref          *iobref;
+        quota_limit_t           limit;
+        int64_t                 space_available;
+        quota_ancestry_built_t  ancestry_cbk;
+        void                   *ancestry_data;
 };
-typedef struct quota_local quota_local_t;
+typedef struct quota_local      quota_local_t;
 
 struct quota_priv {
-        int64_t  timeout;
-        struct list_head limit_head;
+        uint32_t                soft_timeout;
+        uint32_t                hard_timeout;
+        uint32_t               log_timeout;
+        double                 default_soft_lim;
+        gf_boolean_t           is_quota_on;
+        gf_boolean_t           consider_statfs;
+        gf_lock_t              lock;
+        rpc_clnt_prog_t       *quota_enforcer;
+        struct rpcsvc_program *quotad_aggregator;
+        struct rpc_clnt       *rpc_clnt;
+        rpcsvc_t              *rpcsvc;
+        inode_table_t         *itable;
+        char                  *volume_uuid;
+        uint64_t               validation_count;
 };
-typedef struct quota_priv quota_priv_t;
+typedef struct quota_priv      quota_priv_t;
 
-struct limits {
-        struct list_head  limit_list;
-        char             *path;
-        int64_t          value;
-};
-typedef struct limits limits_t;
+int
+quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata, fop_lookup_cbk_t cbk);
+struct rpc_clnt *
+quota_enforcer_init (xlator_t *this, dict_t *options);
 
-uint64_t cn = 1;
+void
+quota_log_usage (xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
+                 int64_t delta);
+
+#endif
diff --git a/xlators/features/quota/src/quotad-aggregator.c b/xlators/features/quota/src/quotad-aggregator.c
new file mode 100644
index 000000000..f3f65ca2a
--- /dev/null
+++ b/xlators/features/quota/src/quotad-aggregator.c
@@ -0,0 +1,423 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "cli1-xdr.h"
+#include "quota.h"
+#include "quotad-helpers.h"
+#include "quotad-aggregator.h"
+
+struct rpcsvc_program quotad_aggregator_prog;
+
+struct iobuf *
+quotad_serialize_reply (rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
+                        xdrproc_t xdrproc)
+{
+        struct iobuf *iob      = NULL;
+        ssize_t       retlen   = 0;
+        ssize_t       xdr_size = 0;
+
+        GF_VALIDATE_OR_GOTO ("server", req, ret);
+
+        /* First, get the io buffer into which the reply in arg will
+         * be serialized.
+         */
+        if (arg && xdrproc) {
+                xdr_size = xdr_sizeof (xdrproc, arg);
+                iob = iobuf_get2 (req->svc->ctx->iobuf_pool, xdr_size);
+                if (!iob) {
+                        gf_log_callingfn (THIS->name, GF_LOG_ERROR,
+                                          "Failed to get iobuf");
+                        goto ret;
+                };
+
+                iobuf_to_iovec (iob, outmsg);
+                /* Use the given serializer to translate the give C structure in arg
+                 * to XDR format which will be written into the buffer in outmsg.
+                 */
+                /* retlen is used to received the error since size_t is unsigned and we
+                 * need -1 for error notification during encoding.
+                 */
+
+                retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
+                if (retlen == -1) {
+                        /* Failed to Encode 'GlusterFS' msg in RPC is not exactly
+                           failure of RPC return values.. client should get
+                           notified about this, so there are no missing frames */
+                        gf_log_callingfn ("", GF_LOG_ERROR, "Failed to encode message");
+                        req->rpc_err = GARBAGE_ARGS;
+                        retlen = 0;
+                }
+        }
+        outmsg->iov_len = retlen;
+ret:
+        if (retlen == -1) {
+                iobuf_unref (iob);
+                iob = NULL;
+        }
+
+        return iob;
+}
+
+int
+quotad_aggregator_submit_reply (call_frame_t *frame, rpcsvc_request_t *req,
+                                void *arg, struct iovec *payload,
+                                int payloadcount, struct iobref *iobref,
+                                xdrproc_t xdrproc)
+{
+        struct iobuf              *iob        = NULL;
+        int                        ret        = -1;
+        struct iovec               rsp        = {0,};
+        quotad_aggregator_state_t *state      = NULL;
+        char                       new_iobref = 0;
+
+        GF_VALIDATE_OR_GOTO ("server", req, ret);
+
+        if (frame) {
+                state = frame->root->state;
+                frame->local = NULL;
+        }
+
+        if (!iobref) {
+                iobref = iobref_new ();
+                if (!iobref) {
+                        goto ret;
+                }
+
+                new_iobref = 1;
+        }
+
+        iob = quotad_serialize_reply (req, arg, &rsp, xdrproc);
+        if (!iob) {
+                gf_log ("", GF_LOG_ERROR, "Failed to serialize reply");
+                goto ret;
+        }
+
+        iobref_add (iobref, iob);
+
+        ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount,
+                                     iobref);
+
+        iobuf_unref (iob);
+
+        ret = 0;
+ret:
+        if (state) {
+                quotad_aggregator_free_state (state);
+        }
+
+        if (frame) {
+                if (frame->root->client)
+                        gf_client_unref (frame->root->client);
+
+                STACK_DESTROY (frame->root);
+        }
+
+        if (new_iobref) {
+                iobref_unref (iobref);
+        }
+
+        return ret;
+}
+
+int
+quotad_aggregator_getlimit_cbk (xlator_t *this, call_frame_t *frame,
+                                void *lookup_rsp)
+{
+        gfs3_lookup_rsp            *rsp   = lookup_rsp;
+        gf_cli_rsp                 cli_rsp = {0,};
+        dict_t                     *xdata = NULL;
+        int                         ret = -1;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
+                                      (rsp->xdata.xdata_val),
+                                      (rsp->xdata.xdata_len), rsp->op_ret,
+                                      rsp->op_errno, out);
+
+        ret = 0;
+out:
+        rsp->op_ret = ret;
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to unserialize "
+                        "nameless lookup rsp");
+                goto reply;
+        }
+        cli_rsp.op_ret = rsp->op_ret;
+        cli_rsp.op_errno = rsp->op_errno;
+        cli_rsp.op_errstr = "";
+        if (xdata) {
+                GF_PROTOCOL_DICT_SERIALIZE (frame->this, xdata,
+                                            (&cli_rsp.dict.dict_val),
+                                            (cli_rsp.dict.dict_len),
+                                            cli_rsp.op_errno, reply);
+        }
+
+reply:
+        quotad_aggregator_submit_reply (frame, frame->local, (void*)&cli_rsp, NULL, 0,
+                                        NULL, (xdrproc_t)xdr_gf_cli_rsp);
+
+        dict_unref (xdata);
+        GF_FREE (cli_rsp.dict.dict_val);
+        return 0;
+}
+
+int
+quotad_aggregator_getlimit (rpcsvc_request_t *req)
+{
+        call_frame_t              *frame = NULL;
+        gf_cli_req                 cli_req = {{0}, };
+        gf_cli_rsp                 cli_rsp = {0};
+        gfs3_lookup_req            args  = {{0,},};
+        gfs3_lookup_rsp            rsp   = {0,};
+        quotad_aggregator_state_t *state = NULL;
+        xlator_t                  *this  = NULL;
+        dict_t                    *dict  = NULL;
+        int                        ret   = -1, op_errno = 0;
+        char                      *gfid_str = NULL;
+        uuid_t                     gfid = {0};
+
+        GF_VALIDATE_OR_GOTO ("quotad-aggregator", req, err);
+
+        this = THIS;
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0)  {
+                //failed to decode msg;
+                gf_log ("", GF_LOG_ERROR, "xdr decoding error");
+                req->rpc_err = GARBAGE_ARGS;
+                goto err;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len, &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto err;
+                }
+        }
+
+        ret = dict_get_str (dict, "gfid", &gfid_str);
+        if (ret) {
+                goto err;
+        }
+
+        uuid_parse ((const char*)gfid_str, gfid);
+
+        frame = quotad_aggregator_get_frame_from_req (req);
+        if (frame == NULL) {
+                rsp.op_errno = ENOMEM;
+                goto err;
+        }
+        state = frame->root->state;
+        state->xdata = dict;
+        ret = dict_set_int32 (state->xdata, QUOTA_LIMIT_KEY, 42);
+        if (ret)
+                goto err;
+
+        ret = dict_set_int32 (state->xdata, QUOTA_SIZE_KEY, 42);
+        if (ret)
+                goto err;
+
+        ret = dict_set_int32 (state->xdata, GET_ANCESTRY_PATH_KEY,42);
+        if (ret)
+                goto err;
+
+        memcpy (&args.gfid, &gfid, 16);
+
+        args.bname           = alloca (req->msg[0].iov_len);
+        args.xdata.xdata_val = alloca (req->msg[0].iov_len);
+
+        ret = qd_nameless_lookup (this, frame, &args, state->xdata,
+                                  quotad_aggregator_getlimit_cbk);
+        if (ret) {
+                rsp.op_errno = ret;
+                goto err;
+        }
+
+        return ret;
+
+err:
+        cli_rsp.op_ret = -1;
+        cli_rsp.op_errno = op_errno;
+        cli_rsp.op_errstr = "";
+
+        quotad_aggregator_getlimit_cbk (this, frame, &cli_rsp);
+        dict_unref (dict);
+
+        return ret;
+}
+
+int
+quotad_aggregator_lookup_cbk (xlator_t *this, call_frame_t *frame,
+                              void *rsp)
+{
+        quotad_aggregator_submit_reply (frame, frame->local, rsp, NULL, 0, NULL,
+                                        (xdrproc_t)xdr_gfs3_lookup_rsp);
+
+        return 0;
+}
+
+
+int
+quotad_aggregator_lookup (rpcsvc_request_t *req)
+{
+        call_frame_t              *frame = NULL;
+        gfs3_lookup_req            args  = {{0,},};
+        int                        ret   = -1, op_errno = 0;
+        gfs3_lookup_rsp            rsp   = {0,};
+        quotad_aggregator_state_t *state = NULL;
+        xlator_t                  *this  = NULL;
+
+        GF_VALIDATE_OR_GOTO ("quotad-aggregator", req, err);
+
+        this = THIS;
+
+        args.bname           = alloca (req->msg[0].iov_len);
+        args.xdata.xdata_val = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_lookup_req);
+        if (ret < 0) {
+                rsp.op_errno = EINVAL;
+                goto err;
+        }
+
+        frame = quotad_aggregator_get_frame_from_req (req);
+        if (frame == NULL) {
+                rsp.op_errno = ENOMEM;
+                goto err;
+        }
+
+        state = frame->root->state;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, state->xdata,
+                                      (args.xdata.xdata_val),
+                                      (args.xdata.xdata_len), ret,
+                                      op_errno, err);
+
+
+        ret = qd_nameless_lookup (this, frame, &args, state->xdata,
+                                  quotad_aggregator_lookup_cbk);
+        if (ret) {
+                rsp.op_errno = ret;
+                goto err;
+        }
+
+        return ret;
+
+err:
+        rsp.op_ret = -1;
+        rsp.op_errno = op_errno;
+
+        quotad_aggregator_lookup_cbk (this, frame, &rsp);
+        return ret;
+}
+
+int
+quotad_aggregator_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                              void *data)
+{
+        if (!xl || !data) {
+                gf_log_callingfn ("server", GF_LOG_WARNING,
+                                  "Calling rpc_notify without initializing");
+                goto out;
+        }
+
+        switch (event) {
+        case RPCSVC_EVENT_ACCEPT:
+                break;
+
+        case RPCSVC_EVENT_DISCONNECT:
+                break;
+
+        default:
+                break;
+        }
+
+out:
+        return 0;
+}
+
+int
+quotad_aggregator_init (xlator_t *this)
+{
+        quota_priv_t *priv = NULL;
+        int           ret  = -1;
+
+        priv = this->private;
+
+        ret = dict_set_str (this->options, "transport.address-family", "unix");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (this->options, "transport-type", "socket");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (this->options, "transport.socket.listen-path",
+                            "/tmp/quotad.socket");
+        if (ret)
+                goto out;
+
+        /* RPC related */
+        priv->rpcsvc = rpcsvc_init (this, this->ctx, this->options, 0);
+        if (priv->rpcsvc == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "creation of rpcsvc failed");
+                ret = -1;
+                goto out;
+        }
+
+        ret = rpcsvc_create_listeners (priv->rpcsvc, this->options,
+                                       this->name);
+        if (ret < 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "creation of listener failed");
+                ret = -1;
+                goto out;
+        }
+
+        priv->quotad_aggregator = &quotad_aggregator_prog;
+        quotad_aggregator_prog.options = this->options;
+
+        ret = rpcsvc_program_register (priv->rpcsvc, &quotad_aggregator_prog);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "registration of program (name:%s, prognum:%d, "
+                        "progver:%d) failed", quotad_aggregator_prog.progname,
+                        quotad_aggregator_prog.prognum,
+                        quotad_aggregator_prog.progver);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+rpcsvc_actor_t quotad_aggregator_actors[] = {
+        [GF_AGGREGATOR_NULL]     = {"NULL", GF_AGGREGATOR_NULL, NULL, NULL, 0,
+                                    DRC_NA},
+        [GF_AGGREGATOR_LOOKUP]   = {"LOOKUP", GF_AGGREGATOR_NULL,
+                                    quotad_aggregator_lookup, NULL, 0, DRC_NA},
+        [GF_AGGREGATOR_GETLIMIT] = {"GETLIMIT", GF_AGGREGATOR_GETLIMIT,
+                                   quotad_aggregator_getlimit, NULL, 0},
+};
+
+
+struct rpcsvc_program quotad_aggregator_prog = {
+        .progname  = "GlusterFS 3.3",
+        .prognum   = GLUSTER_AGGREGATOR_PROGRAM,
+        .progver   = GLUSTER_AGGREGATOR_VERSION,
+        .numactors = GF_AGGREGATOR_MAXVALUE,
+        .actors    = quotad_aggregator_actors
+};
diff --git a/xlators/features/quota/src/quotad-aggregator.h b/xlators/features/quota/src/quotad-aggregator.h
new file mode 100644
index 000000000..5ddea5b3c
--- /dev/null
+++ b/xlators/features/quota/src/quotad-aggregator.h
@@ -0,0 +1,37 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _QUOTAD_AGGREGATOR_H
+#define _QUOTAD_AGGREGATOR_H
+
+#include "quota.h"
+#include "stack.h"
+#include "glusterfs3-xdr.h"
+#include "inode.h"
+
+typedef struct {
+        void          *pool;
+        xlator_t      *this;
+	xlator_t      *active_subvol;
+        inode_table_t *itable;
+        loc_t          loc;
+        dict_t        *xdata;
+} quotad_aggregator_state_t;
+
+typedef int (*quotad_aggregator_lookup_cbk_t) (xlator_t *this,
+                                               call_frame_t *frame,
+                                               void *rsp);
+int
+qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req,
+                    dict_t *xdata, quotad_aggregator_lookup_cbk_t lookup_cbk);
+int
+quotad_aggregator_init (xlator_t *this);
+
+#endif
diff --git a/xlators/features/quota/src/quotad-helpers.c b/xlators/features/quota/src/quotad-helpers.c
new file mode 100644
index 000000000..fd3099114
--- /dev/null
+++ b/xlators/features/quota/src/quotad-helpers.c
@@ -0,0 +1,113 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "quotad-helpers.h"
+
+quotad_aggregator_state_t *
+get_quotad_aggregator_state (xlator_t *this, rpcsvc_request_t *req)
+{
+        quotad_aggregator_state_t *state         = NULL;
+	xlator_t                  *active_subvol = NULL;
+        quota_priv_t              *priv          = NULL;
+
+        state = (void *)GF_CALLOC (1, sizeof (*state),
+                                   gf_quota_mt_aggregator_state_t);
+        if (!state)
+                return NULL;
+
+	state->this = THIS;
+        priv = this->private;
+
+        LOCK (&priv->lock);
+        {
+                active_subvol = state->active_subvol = FIRST_CHILD (this);
+        }
+        UNLOCK (&priv->lock);
+
+        if (active_subvol->itable == NULL)
+                active_subvol->itable = inode_table_new (4096, active_subvol);
+
+	state->itable = active_subvol->itable;
+
+        state->pool = this->ctx->pool;
+
+        return state;
+}
+
+void
+quotad_aggregator_free_state (quotad_aggregator_state_t *state)
+{
+        if (state->xdata)
+                dict_unref (state->xdata);
+
+        GF_FREE (state);
+}
+
+call_frame_t *
+quotad_aggregator_alloc_frame (rpcsvc_request_t *req)
+{
+        call_frame_t              *frame = NULL;
+        quotad_aggregator_state_t *state = NULL;
+        xlator_t                  *this  = NULL;
+
+        GF_VALIDATE_OR_GOTO ("server", req, out);
+        GF_VALIDATE_OR_GOTO ("server", req->trans, out);
+        GF_VALIDATE_OR_GOTO ("server", req->svc, out);
+        GF_VALIDATE_OR_GOTO ("server", req->svc->ctx, out);
+
+        this = req->svc->mydata;
+
+        frame = create_frame (this, req->svc->ctx->pool);
+        if (!frame)
+                goto out;
+
+        state = get_quotad_aggregator_state (this, req);
+        if (!state)
+                goto out;
+
+        frame->root->state = state;
+        frame->root->unique = 0;
+
+        frame->this = this;
+out:
+        return frame;
+}
+
+call_frame_t *
+quotad_aggregator_get_frame_from_req (rpcsvc_request_t *req)
+{
+        call_frame_t *frame  = NULL;
+        client_t     *client = NULL;
+
+        GF_VALIDATE_OR_GOTO ("server", req, out);
+
+        frame = quotad_aggregator_alloc_frame (req);
+        if (!frame)
+                goto out;
+
+        client = req->trans->xl_private;
+
+        frame->root->op       = req->procnum;
+
+        frame->root->unique   = req->xid;
+
+        frame->root->uid      = req->uid;
+        frame->root->gid      = req->gid;
+        frame->root->pid      = req->pid;
+
+        gf_client_ref (client);
+        frame->root->client   = client;
+
+        frame->root->lk_owner = req->lk_owner;
+
+        frame->local = req;
+out:
+        return frame;
+}
diff --git a/xlators/features/quota/src/quotad-helpers.h b/xlators/features/quota/src/quotad-helpers.h
new file mode 100644
index 000000000..a10fb7fa8
--- /dev/null
+++ b/xlators/features/quota/src/quotad-helpers.h
@@ -0,0 +1,24 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef QUOTAD_HELPERS_H
+#define QUOTAD_HELPERS_H
+
+#include "rpcsvc.h"
+#include "quota.h"
+#include "quotad-aggregator.h"
+
+void
+quotad_aggregator_free_state (quotad_aggregator_state_t *state);
+
+call_frame_t *
+quotad_aggregator_get_frame_from_req (rpcsvc_request_t *req);
+
+#endif
diff --git a/xlators/features/quota/src/quotad.c b/xlators/features/quota/src/quotad.c
new file mode 100644
index 000000000..243b943e9
--- /dev/null
+++ b/xlators/features/quota/src/quotad.c
@@ -0,0 +1,210 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include "quota.h"
+#include "quotad-aggregator.h"
+#include "common-utils.h"
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_quota_mt_end + 1);
+
+        if (0 != ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting "
+                        "init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+qd_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        quotad_aggregator_lookup_cbk_t  lookup_cbk = NULL;
+        gfs3_lookup_rsp                 rsp = {0, };
+
+        lookup_cbk = cookie;
+
+        rsp.op_ret = op_ret;
+        rsp.op_errno = op_errno;
+
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val),
+                                    rsp.xdata.xdata_len, rsp.op_errno, out);
+
+        gf_stat_from_iatt (&rsp.stat, buf);
+
+out:
+        lookup_cbk (this, frame, &rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        inode_unref (inode);
+
+        return 0;
+}
+
+xlator_t *
+qd_find_subvol (xlator_t *this, char *volume_uuid)
+{
+        xlator_list_t *child  = NULL;
+        xlator_t      *subvol = NULL;
+        char           key[1024];
+        char          *optstr = NULL;
+
+        if (!this || !volume_uuid)
+                goto out;
+
+        for (child = this->children; child; child = child->next) {
+                snprintf(key, 1024, "%s.volume-id", child->xlator->name);
+                if (dict_get_str(this->options, key, &optstr) < 0)
+                        continue;
+
+                if (strcmp (optstr, volume_uuid) == 0) {
+                        subvol = child->xlator;
+                        break;
+                }
+        }
+
+out:
+        return subvol;
+}
+
+int
+qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req,
+                    dict_t *xdata, quotad_aggregator_lookup_cbk_t lookup_cbk)
+{
+        gfs3_lookup_rsp            rsp         = {0, };
+        int                        op_errno    = 0, ret = -1;
+        loc_t                      loc         = {0, };
+        quotad_aggregator_state_t *state       = NULL;
+        quota_priv_t              *priv        = NULL;
+        xlator_t                  *subvol      = NULL;
+        char                      *volume_uuid = NULL;
+
+        priv = this->private;
+        state = frame->root->state;
+
+        frame->root->op = GF_FOP_LOOKUP;
+
+        loc.inode = inode_new (state->itable);
+        if (loc.inode == NULL) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        memcpy (loc.gfid, req->gfid, 16);
+
+        ret = dict_get_str (xdata, "volume-uuid", &volume_uuid);
+        if (ret < 0) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        subvol = qd_find_subvol (this, volume_uuid);
+        if (subvol == NULL) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        STACK_WIND_COOKIE (frame, qd_lookup_cbk, lookup_cbk, subvol,
+                           subvol->fops->lookup, &loc, xdata);
+        return 0;
+
+out:
+        rsp.op_ret = -1;
+        rsp.op_errno = op_errno;
+
+        lookup_cbk (this, frame, &rsp);
+
+        inode_unref (loc.inode);
+        return 0;
+}
+
+int
+qd_reconfigure (xlator_t *this, dict_t *options)
+{
+        /* As of now quotad is restarted upon alteration of volfile */
+        return 0;
+}
+
+void
+qd_fini (xlator_t *this)
+{
+        return;
+}
+
+int32_t
+qd_init (xlator_t *this)
+{
+        int32_t          ret            = -1;
+        quota_priv_t    *priv           = NULL;
+
+        if (NULL == this->children) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "FATAL: quota (%s) not configured for min of 1 child",
+                        this->name);
+                ret = -1;
+                goto err;
+        }
+
+        QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err);
+        LOCK_INIT (&priv->lock);
+
+        this->private = priv;
+
+        ret = quotad_aggregator_init (this);
+        if (ret < 0)
+                goto err;
+
+        ret = 0;
+err:
+        if (ret) {
+                GF_FREE (priv);
+        }
+        return ret;
+}
+
+class_methods_t class_methods = {
+        .init           = qd_init,
+        .fini           = qd_fini,
+        .reconfigure    = qd_reconfigure,
+};
+
+struct xlator_fops fops = {
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key   = {"transport-type"},
+          .value = {"rpc", "rpc-over-rdma", "tcp", "socket", "ib-verbs",
+                    "unix", "ib-sdp", "tcp/server", "ib-verbs/server", "rdma",
+                    "rdma*([ \t]),*([ \t])socket",
+                    "rdma*([ \t]),*([ \t])tcp",
+                    "tcp*([ \t]),*([ \t])rdma",
+                    "socket*([ \t]),*([ \t])rdma"},
+          .type  = GF_OPTION_TYPE_STR
+        },
+        { .key   = {"transport.*"},
+          .type  = GF_OPTION_TYPE_ANY,
+        },
+        {.key = {NULL}}
+};
diff --git a/xlators/features/read-only/src/Makefile.am b/xlators/features/read-only/src/Makefile.am
index 31ae4f340..4c1462137 100644
--- a/xlators/features/read-only/src/Makefile.am
+++ b/xlators/features/read-only/src/Makefile.am
@@ -4,18 +4,19 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 noinst_HEADERS = read-only-common.h
 
-read_only_la_LDFLAGS = -module -avoidversion
+read_only_la_LDFLAGS = -module -avoid-version
 
 read_only_la_SOURCES = read-only.c read-only-common.c
 read_only_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
-worm_la_LDFLAGS = -module -avoidversion
+worm_la_LDFLAGS = -module -avoid-version
 
 worm_la_SOURCES = read-only-common.c worm.c
 worm_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/read-only/src/read-only-common.c b/xlators/features/read-only/src/read-only-common.c
index efd175081..56a7a7176 100644
--- a/xlators/features/read-only/src/read-only-common.c
+++ b/xlators/features/read-only/src/read-only-common.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -27,220 +17,223 @@
 
 int32_t
 ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict)
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (xattrop, frame, -1, EROFS, NULL);
+        STACK_UNWIND_STRICT (xattrop, frame, -1, EROFS, NULL, xdata);
         return 0;
 }
 
 int32_t
 ro_fxattrop (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fxattrop, frame, -1, EROFS, NULL);
+        STACK_UNWIND_STRICT (fxattrop, frame, -1, EROFS, NULL, xdata);
         return 0;
 }
 
 int32_t
 ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
             loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type)
+            entrylk_type type, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (entrylk, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (entrylk, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type type)
+             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fentrylk, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (fentrylk, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock)
+            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (inodelk, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (inodelk, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock)
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (finodelk, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (finodelk, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock)
+       struct gf_flock *flock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (lk, frame, -1, EROFS, NULL);
+        STACK_UNWIND_STRICT (lk, frame, -1, EROFS, NULL, xdata);
         return 0;
 }
 
 int32_t
 ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid)
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setattr, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (setattr, frame, -1, EROFS, NULL, NULL, xdata);
 	return 0;
 }
 
 int32_t
 ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid)
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (fsetattr, frame, -1, EROFS, NULL, NULL, xdata);
         return 0;
 }
 
 
 int32_t
-ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (truncate, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, EROFS, NULL, NULL, xdata);
 	return 0;
 }
 
 int32_t
-ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, EROFS, NULL, NULL, xdata);
 	return 0;
 }
 
 int
 ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, dict_t *params)
+          dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (mknod, frame, -1, EROFS, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (mknod, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
         return 0;
 }
 
 
 int
 ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dict_t *params)
+          mode_t umask, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (mkdir, frame, -1, EROFS, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (mkdir, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
         return 0;
 }
 
 int32_t
-ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (unlink, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (unlink, frame, -1, EROFS, NULL, NULL, xdata);
         return 0;
 }
 
 
 int
-ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rmdir, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (rmdir, frame, -1, EROFS, NULL, NULL, xdata);
         return 0;
 }
 
 
 int
 ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, dict_t *params)
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (symlink, frame, -1, EROFS, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (symlink, frame, -1, EROFS, NULL, NULL, NULL,
+                             NULL, xdata);
         return 0;
 }
 
 
 
 int32_t
-ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (rename, frame, -1, EROFS, NULL, NULL, NULL, NULL,
-                             NULL);
+                             NULL, xdata);
         return 0;
 }
 
 
 int32_t
-ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (link, frame, -1, EROFS, NULL, NULL, NULL, NULL);
+        STACK_UNWIND_STRICT (link, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
         return 0;
 }
 
 int32_t
 ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params)
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (create, frame, -1, EROFS, NULL, NULL, NULL,
-                             NULL, NULL);
+                             NULL, NULL, xdata);
         return 0;
 }
 
 
 static int32_t
 ro_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd)
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 	return 0;
 }
 
 int32_t
 ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+         fd_t *fd, dict_t *xdata)
 {
         if (((flags & O_ACCMODE) == O_WRONLY) ||
               ((flags & O_ACCMODE) == O_RDWR)) {
-                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, xdata);
                 return 0;
 	}
 
 	STACK_WIND (frame, ro_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
 	return 0;
 }
 
 int32_t
 ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags)
+              int32_t flags, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (fsetxattr, frame, -1, EROFS, xdata);
 	return 0;
 }
 
 int32_t
-ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsyncdir, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (fsyncdir, frame, -1, EROFS, xdata);
 	return 0;
 }
 
 int32_t
 ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref)
+           int32_t count, off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (writev, frame, -1, EROFS, NULL, NULL);
+        STACK_UNWIND_STRICT (writev, frame, -1, EROFS, NULL, NULL, xdata);
         return 0;
 }
 
 
 int32_t
 ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags)
+             int32_t flags, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (setxattr, frame, -1, EROFS, xdata);
         return 0;
 }
 
 int32_t
 ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name)
+                const char *name, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (removexattr, frame, -1, EROFS);
+        STACK_UNWIND_STRICT (removexattr, frame, -1, EROFS, xdata);
         return 0;
 }
diff --git a/xlators/features/read-only/src/read-only-common.h b/xlators/features/read-only/src/read-only-common.h
index 8c89e88ce..5d4c7e260 100644
--- a/xlators/features/read-only/src/read-only-common.h
+++ b/xlators/features/read-only/src/read-only-common.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -27,97 +17,99 @@
 
 int32_t
 ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict);
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
 
 int32_t
 ro_fxattrop (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict);
+             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
 
 int32_t
 ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
             loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type);
+            entrylk_type type, dict_t *xdata);
 
 int32_t
 ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
              fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type
-             type);
+             type, dict_t *xdata);
 
 int32_t
 ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock);
+            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
 ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock);
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
 ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock);
+       struct gf_flock *flock, dict_t *xdata);
 
 int32_t
 ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid);
+            struct iatt *stbuf, int32_t valid, dict_t *xdata);
 
 int32_t
 ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid);
+             struct iatt *stbuf, int32_t valid, dict_t *xdata);
 
 
 int32_t
-ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset);
+ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata);
 
 int32_t
-ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset);
+ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata);
 
 int
 ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, dict_t *params);
+          dev_t rdev, mode_t umask, dict_t *xdata);
 
 int
 ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dict_t *params);
+          mode_t umask, dict_t *xdata);
 
 int32_t
-ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc);
+ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
 
 int
-ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags);
+ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
 
 
 int
 ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, dict_t *params);
+            loc_t *loc, mode_t umask, dict_t *xdata);
 
 int32_t
-ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc);
+ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
-ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc);
+ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
 ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params);
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
 ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags);
+         fd_t *fd, dict_t *xdata);
 
 int32_t
 ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags);
+              int32_t flags, dict_t *xdata);
 
 int32_t
-ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags);
+ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata);
 
 int32_t
 ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref);
+           int32_t count, off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata);
 
 int32_t
 ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags);
+             int32_t flags, dict_t *xdata);
 
 int32_t
 ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name);
+                const char *name, dict_t *xdata);
diff --git a/xlators/features/read-only/src/read-only.c b/xlators/features/read-only/src/read-only.c
index b11e84f24..e49e54a1b 100644
--- a/xlators/features/read-only/src/read-only.c
+++ b/xlators/features/read-only/src/read-only.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/features/read-only/src/worm.c b/xlators/features/read-only/src/worm.c
index 790bc3234..16c3eb3da 100644
--- a/xlators/features/read-only/src/worm.c
+++ b/xlators/features/read-only/src/worm.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -28,25 +18,25 @@
 
 static int32_t
 worm_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd)
+               int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
 int32_t
 worm_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+           fd_t *fd, dict_t *xdata)
 {
         if ((((flags & O_ACCMODE) == O_WRONLY) ||
               ((flags & O_ACCMODE) == O_RDWR)) &&
               !(flags & O_APPEND)) {
-                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, NULL);
                 return 0;
         }
 
         STACK_WIND (frame, worm_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
         return 0;
 }
 
@@ -75,7 +65,6 @@ fini (xlator_t *this)
 }
 
 struct xlator_fops fops = {
-        
         .open        = worm_open,
 
         .unlink      = ro_unlink,
@@ -92,8 +81,7 @@ struct xlator_fops fops = {
         .lk          = ro_lk,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
         { .key = {NULL} },
diff --git a/xlators/features/trash/src/Makefile.am b/xlators/features/trash/src/Makefile.am
index 4671d06d3..5251eb082 100644
--- a/xlators/features/trash/src/Makefile.am
+++ b/xlators/features/trash/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = trash.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-trash_la_LDFLAGS = -module -avoidversion 
+trash_la_LDFLAGS = -module -avoid-version 
 
 trash_la_SOURCES = trash.c
 trash_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = trash.h trash-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/trash/src/trash-mem-types.h b/xlators/features/trash/src/trash-mem-types.h
index bb2cd33cf..0e6ef572f 100644
--- a/xlators/features/trash/src/trash-mem-types.h
+++ b/xlators/features/trash/src/trash-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __TRASH_MEM_TYPES_H__
 #define __TRASH_MEM_TYPES_H__
 
diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c
index 15d9a429f..addeb66a0 100644
--- a/xlators/features/trash/src/trash.c
+++ b/xlators/features/trash/src/trash.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -170,8 +160,7 @@ trash_unlink_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         GF_FREE (cookie);
-        if (tmp_str)
-                GF_FREE (tmp_str);
+        GF_FREE (tmp_str);
 
         return 0;
 }
@@ -442,8 +431,7 @@ trash_rename_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
-        if (tmp_str)
-                GF_FREE (tmp_str);
+        GF_FREE (tmp_str);
 
         return 0;
 }
@@ -504,9 +492,7 @@ trash_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         trash_elim_pattern_t  *trav  = NULL;
         trash_private_t *priv = NULL;
         trash_local_t   *local = NULL;
-        struct tm       *tm = NULL;
-        char             timestr[256] = {0,};
-        time_t           utime = 0;
+        char             timestr[64] = {0,};
         int32_t          match = 0;
 
         priv = this->private;
@@ -553,9 +539,8 @@ trash_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         {
                 /* append timestamp to file name */
                 /* TODO: can we make it optional? */
-                utime = time (NULL);
-                tm    = localtime (&utime);
-                strftime (timestr, 256, ".%Y-%m-%d-%H%M%S", tm);
+                gf_time_ftm (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
                 strcat (local->newpath, timestr);
         }
 
@@ -574,9 +559,7 @@ trash_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         trash_elim_pattern_t  *trav  = NULL;
         trash_private_t *priv = NULL;
         trash_local_t   *local = NULL;
-        struct tm       *tm = NULL;
-        char             timestr[256] = {0,};
-        time_t           utime = 0;
+        char             timestr[64] = {0,};
         int32_t          match = 0;
 
         priv = this->private;
@@ -625,9 +608,8 @@ trash_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         {
                 /* append timestamp to file name */
                 /* TODO: can we make it optional? */
-                utime = time (NULL);
-                tm    = localtime (&utime);
-                strftime (timestr, 256, ".%Y-%m-%d-%H%M%S", tm);
+                gf_time_fmt (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
                 strcat (local->newpath, timestr);
         }
 
@@ -930,8 +912,7 @@ trash_truncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
-        if (tmp_str)
-                GF_FREE (tmp_str);
+        GF_FREE (tmp_str);
 
         return 0;
 }
@@ -943,10 +924,8 @@ trash_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         trash_private_t     *priv  = NULL;
         trash_local_t       *local = NULL;
-        struct tm           *tm = NULL;
-        char                 timestr[256] = {0,};
+        char                 timestr[64] = {0,};
         char                 loc_newname[PATH_MAX] = {0,};
-        time_t               utime = 0;
         int32_t              flags = 0;
 
         priv = this->private;
@@ -978,9 +957,8 @@ trash_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         strcat (local->newpath, local->loc.path);
 
         {
-                utime = time (NULL);
-                tm    = localtime (&utime);
-                strftime (timestr, 256, ".%Y-%m-%d-%H%M%S", tm);
+                gf_time_fmt (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
                 strcat (local->newpath, timestr);
         }
         strcpy (loc_newname,local->loc.name);
@@ -1296,8 +1274,7 @@ trash_ftruncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
-        if (tmp_str)
-                GF_FREE (tmp_str);
+        GF_FREE (tmp_str);
 
         return 0;
 }
@@ -1346,11 +1323,9 @@ trash_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         trash_private_t       *priv = NULL;
         trash_local_t         *local = NULL;
         dentry_t              *dir_entry = NULL;
-        struct tm             *tm = NULL;
         char                  *pathbuf = NULL;
         inode_t               *newinode = NULL;
-        time_t                 utime = 0;
-        char                   timestr[256];
+        char                   timestr[64];
         int32_t                retval = 0;
         int32_t                match = 0;
 
@@ -1389,10 +1364,7 @@ trash_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 return 0;
         }
 
-        utime = time (NULL);
-        tm    = localtime (&utime);
-        strftime (timestr, 256, ".%Y-%m-%d-%H%M%S", tm);
-
+        gf_time_fmt (timestr, sizeof timestr, time (NULL), gf_timefmt_F_HMS);
         strcpy (local->newpath, priv->trash_dir);
         strcat (local->newpath, pathbuf);
         strcat (local->newpath, timestr);
@@ -1536,8 +1508,7 @@ fini (xlator_t *this)
         trash_private_t *priv = NULL;
 
         priv = this->private;
-        if (priv)
-                GF_FREE (priv);
+        GF_FREE (priv);
 
         return;
 }
diff --git a/xlators/features/trash/src/trash.h b/xlators/features/trash/src/trash.h
index d385ee346..9a7c03361 100644
--- a/xlators/features/trash/src/trash.h
+++ b/xlators/features/trash/src/trash.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __TRASH_H__
 #define __TRASH_H__
 
diff --git a/xlators/lib/src/libxlator.c b/xlators/lib/src/libxlator.c
index 85b5f685e..4e680c510 100644
--- a/xlators/lib/src/libxlator.c
+++ b/xlators/lib/src/libxlator.c
@@ -1,25 +1,44 @@
-/*Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include "mem-types.h"
 #include "libxlator.h"
 
 
+int marker_xtime_default_gauge[] = {
+        [MCNT_FOUND]    =  1,
+        [MCNT_NOTFOUND] = -1,
+        [MCNT_ENODATA]  = -1,
+        [MCNT_ENOTCONN] = -1,
+        [MCNT_ENOENT]   = -1,
+        [MCNT_EOTHER]   = -1,
+};
+
+int marker_uuid_default_gauge[] = {
+        [MCNT_FOUND]    =  1,
+        [MCNT_NOTFOUND] =  0,
+        [MCNT_ENODATA]  =  0,
+        [MCNT_ENOTCONN] =  0,
+        [MCNT_ENOENT]   =  0,
+        [MCNT_EOTHER]   =  0,
+};
+
+static int marker_idx_errno_map[] = {
+        [MCNT_FOUND]    = EINVAL,
+        [MCNT_NOTFOUND] = EINVAL,
+        [MCNT_ENOENT]   = ENOENT,
+        [MCNT_ENOTCONN] = ENOTCONN,
+        [MCNT_ENODATA]  = ENODATA,
+        [MCNT_EOTHER]   = EINVAL,
+        [MCNT_MAX]      = 0,
+};
+
 /*Copy the contents of oldtimebuf to newtimbuf*/
 static void
 update_timebuf (uint32_t *oldtimbuf, uint32_t *newtimebuf)
@@ -56,28 +75,67 @@ match_uuid_local (const char *name, char *uuid)
 static void
 marker_local_incr_errcount (xl_marker_local_t *local, int op_errno)
 {
+        marker_result_idx_t i = -1;
+
         if (!local)
                 return;
 
         switch (op_errno) {
                 case ENODATA:
-                        local->enodata_count++;
+                        i = MCNT_ENODATA;
                         break;
                 case ENOENT:
-                        local->enoent_count++;
+                        i = MCNT_ENOENT;
                         break;
                 case ENOTCONN:
-                        local->enotconn_count++;
+                        i = MCNT_ENOTCONN;
                         break;
                 default:
+                        i = MCNT_EOTHER;
                         break;
         }
+
+        local->count[i]++;
+}
+
+static int
+evaluate_marker_results (int *gauge, int *count)
+{
+        int i = 0;
+        int op_errno = 0;
+        gf_boolean_t sane = _gf_true;
+
+        /* check if the policy of the gauge is violated;
+         * if yes, try to get the best errno, ie. look
+         * for the first position where there is a more
+         * specific kind of vioilation than the generic EINVAL
+         */
+        for (i = 0; i < MCNT_MAX; i++) {
+                if (sane) {
+                        if ((gauge[i] > 0 && count[i] < gauge[i]) ||
+                            (gauge[i] < 0 && count[i] >= -gauge[i])) {
+                                sane = _gf_false;
+                                /* generic action: adopt corresponding errno */
+                                op_errno = marker_idx_errno_map[i];
+                        }
+                } else {
+                        /* already insane; trying to get a more informative
+                         * errno by checking subsequent counters
+                         */
+                        if (count[i] > 0)
+                                op_errno = marker_idx_errno_map[i];
+                }
+                if (op_errno && op_errno != EINVAL)
+                        break;
+        }
+
+        return op_errno;
 }
 
 /* Aggregate all the <volid>.xtime attrs of the cluster and send the max*/
 int32_t
 cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict)
+                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
 {
 
@@ -107,14 +165,10 @@ cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         {
                 callcnt = --local->call_count;
 
-                if (local->esomerr)
-                        goto unlock;
-
                 vol_uuid = local->vol_uuid;
 
                 if (op_ret) {
                         marker_local_incr_errcount (local, op_errno);
-                        local->esomerr = op_errno;
                         goto unlock;
                 }
 
@@ -128,11 +182,11 @@ cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (dict_get_ptr (dict, marker_xattr, (void **)&net_timebuf)) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Unable to get <uuid>.xtime attr");
-                        local->noxtime_count++;
+                        local->count[MCNT_NOTFOUND]++;
                         goto unlock;
                 }
 
-                if (local->has_xtime) {
+                if (local->count[MCNT_FOUND]) {
                         get_hosttime (net_timebuf, host_timebuf);
                         if ( (host_timebuf[0]>local->host_timebuf[0]) ||
                                 (host_timebuf[0] == local->host_timebuf[0] &&
@@ -144,7 +198,7 @@ cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 } else {
                         get_hosttime (net_timebuf, local->host_timebuf);
                         update_timebuf (net_timebuf, local->net_timebuf);
-                        local->has_xtime = _gf_true;
+                        local->count[MCNT_FOUND]++;
                 }
 
         }
@@ -156,7 +210,7 @@ unlock:
                 op_errno = 0;
                 need_unwind = 1;
 
-                if (local->has_xtime) {
+                if (local->count[MCNT_FOUND]) {
                         if (!dict)
                                 dict = dict_new();
 
@@ -169,36 +223,30 @@ unlock:
                         }
                 }
 
-                if (local->noxtime_count)
-                        goto out;
-
-                if (local->enodata_count || local->enotconn_count ||
-                                              local->enoent_count) {
+                op_errno = evaluate_marker_results (local->gauge, local->count);
+                if (op_errno)
                         op_ret = -1;
-                        op_errno = local->enodata_count? ENODATA:
-                                        local->enotconn_count? ENOTCONN:
-                                        local->enoent_count? ENOENT:
-                                        local->esomerr;
-                }
         }
 
 out:
         if (need_unwind && local && local->xl_specf_unwind) {
                 frame->local = local->xl_local;
                 local->xl_specf_unwind (frame, op_ret,
-                                         op_errno, dict);
-                return 0;
+                                        op_errno, dict, xdata);
+		GF_FREE (local);
         } else if (need_unwind) {
-                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
+                                     dict, xdata);
         }
 
+        GF_FREE (marker_xattr);
         return 0;
 
 }
 
 int32_t
 cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict)
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
         int32_t             callcnt     = 0;
         struct volume_mark  *volmark    = NULL;
@@ -236,7 +284,7 @@ cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (ret)
                         goto unlock;
 
-                if (marker_has_volinfo (local)) {
+                if (local->count[MCNT_FOUND]) {
                         if ((local->volmark->major != volmark->major) ||
                             (local->volmark->minor != volmark->minor)) {
                                 op_ret = -1;
@@ -265,6 +313,7 @@ cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         uuid_unparse (volmark->uuid, vol_uuid);
                         if (volmark->retval)
                                 local->retval = volmark->retval;
+                        local->count[MCNT_FOUND]++;
                 }
         }
 unlock:
@@ -275,7 +324,7 @@ unlock:
                 op_errno = 0;
                 need_unwind = 1;
 
-                if (marker_has_volinfo (local)) {
+                if (local->count[MCNT_FOUND]) {
                         if (!dict)
                                 dict = dict_new();
 
@@ -285,21 +334,22 @@ unlock:
                                 op_ret = -1;
                                 op_errno = ENOMEM;
                         }
-                } else {
-                        op_ret = -1;
-                        op_errno = local->enotconn_count? ENOTCONN:
-                               local->enoent_count? ENOENT:EINVAL;
                 }
+                op_errno = evaluate_marker_results (local->gauge, local->count);
+                if (op_errno)
+                        op_ret = -1;
         }
 
  out:
         if (need_unwind && local && local->xl_specf_unwind) {
                 frame->local = local->xl_local;
                 local->xl_specf_unwind (frame, op_ret,
-                                        op_errno, dict);
+                                        op_errno, dict, xdata);
+		GF_FREE (local);
                 return 0;
         } else if (need_unwind){
-                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
+                                     dict, xdata);
         }
         return 0;
 }
@@ -310,7 +360,7 @@ cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
                        const char *name, void *xl_local,
                        xlator_specf_unwind_t xl_specf_getxattr_unwind,
                        xlator_t **sub_volumes, int count, int type,
-                       char *vol_uuid)
+                       int *gauge, char *vol_uuid)
 {
         int                i     = 0;
         xl_marker_local_t  *local = NULL;
@@ -333,6 +383,7 @@ cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
         local->call_count = count;
         local->xl_specf_unwind = xl_specf_getxattr_unwind;
         local->vol_uuid = vol_uuid;
+        memcpy (local->gauge, gauge, sizeof (local->gauge));
 
         frame->local = local;
 
@@ -341,12 +392,12 @@ cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
                         STACK_WIND (frame, cluster_markeruuid_cbk,
                                     *(sub_volumes + i),
                                     (*(sub_volumes + i))->fops->getxattr,
-                                    loc, name);
+                                    loc, name, NULL);
                 else if (MARKER_XTIME_TYPE == type)
                         STACK_WIND (frame, cluster_markerxtime_cbk,
                                     *(sub_volumes + i),
                                     (*(sub_volumes + i))->fops->getxattr,
-                                    loc, name);
+                                    loc, name, NULL);
                 else {
                         gf_log (this->name, GF_LOG_WARNING,
                                  "Unrecognized type (%d) of marker attr "
@@ -354,7 +405,7 @@ cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
                         STACK_WIND (frame, default_getxattr_cbk,
                                     *(sub_volumes + i),
                                     (*(sub_volumes + i))->fops->getxattr,
-                                    loc, name);
+                                    loc, name, NULL);
                         break;
                 }
         }
@@ -364,3 +415,113 @@ err:
         return -1;
 
 }
+
+int
+gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value)
+{
+        int ret = -1;
+        uint32_t *net_timebuf = NULL;
+        uint32_t *value_timebuf = NULL;
+        uint32_t host_timebuf[2] = {0,};
+        uint32_t host_value_timebuf[2] = {0,};
+
+        /* stime should be minimum of all the other nodes */
+        ret = dict_get_bin (dst, key, (void **)&net_timebuf);
+        if (ret < 0) {
+                net_timebuf = GF_CALLOC (1, sizeof (int64_t),
+                                           gf_common_mt_char);
+                if (!net_timebuf)
+                        goto out;
+
+                ret = dict_set_bin (dst, key, net_timebuf, sizeof (int64_t));
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "key=%s: dict set failed", key);
+                        goto error;
+                }
+        }
+
+        value_timebuf = data_to_bin (value);
+        if (!value_timebuf) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "key=%s: getting value of stime failed", key);
+                ret = -1;
+                goto out;
+        }
+
+        get_hosttime (value_timebuf, host_value_timebuf);
+        get_hosttime (net_timebuf, host_timebuf);
+
+        /* can't use 'min()' macro here as we need to compare two fields
+           in the array, selectively */
+        if ((host_value_timebuf[0] < host_timebuf[0]) ||
+            ((host_value_timebuf[0] == host_timebuf[0]) &&
+             (host_value_timebuf[1] < host_timebuf[1]))) {
+                update_timebuf (value_timebuf, net_timebuf);
+        }
+
+        ret = 0;
+out:
+        return ret;
+error:
+        /* To be used only when net_timebuf is not set in the dict */
+        if (net_timebuf)
+                GF_FREE (net_timebuf);
+
+        return ret;
+}
+
+int
+gf_get_max_stime (xlator_t *this, dict_t *dst, char *key, data_t *value)
+{
+        int ret = -1;
+        uint32_t *net_timebuf = NULL;
+        uint32_t *value_timebuf = NULL;
+        uint32_t host_timebuf[2] = {0,};
+        uint32_t host_value_timebuf[2] = {0,};
+
+        /* stime should be maximum of all the other nodes */
+        ret = dict_get_bin (dst, key, (void **)&net_timebuf);
+        if (ret < 0) {
+                net_timebuf = GF_CALLOC (1, sizeof (int64_t),
+                                           gf_common_mt_char);
+                if (!net_timebuf)
+                        goto out;
+
+                ret = dict_set_bin (dst, key, net_timebuf, sizeof (int64_t));
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "key=%s: dict set failed", key);
+                        goto error;
+                }
+        }
+
+        value_timebuf = data_to_bin (value);
+        if (!value_timebuf) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "key=%s: getting value of stime failed", key);
+                ret = -1;
+                goto out;
+        }
+
+        get_hosttime (value_timebuf, host_value_timebuf);
+        get_hosttime (net_timebuf, host_timebuf);
+
+        /* can't use 'max()' macro here as we need to compare two fields
+           in the array, selectively */
+        if ((host_value_timebuf[0] > host_timebuf[0]) ||
+            ((host_value_timebuf[0] == host_timebuf[0]) &&
+             (host_value_timebuf[1] > host_timebuf[1]))) {
+                update_timebuf (value_timebuf, net_timebuf);
+        }
+
+        ret = 0;
+out:
+        return ret;
+error:
+        /* To be used only when net_timebuf is not set in the dict */
+        if (net_timebuf)
+                GF_FREE (net_timebuf);
+
+        return ret;
+}
diff --git a/xlators/lib/src/libxlator.h b/xlators/lib/src/libxlator.h
index 1bc20ac6b..175d3141d 100644
--- a/xlators/lib/src/libxlator.h
+++ b/xlators/lib/src/libxlator.h
@@ -1,21 +1,12 @@
-/*Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _LIBXLATOR_H
 #define _LIBXLATOR_H
 
@@ -41,10 +32,12 @@
 #define MARKER_UUID_TYPE    1
 #define MARKER_XTIME_TYPE   2
 #define GF_XATTR_QUOTA_SIZE_KEY "trusted.glusterfs.quota.size"
+#define GF_XATTR_QUOTA_LIMIT_LIST "trusted.limit.list"
 
 
 typedef int32_t (*xlator_specf_unwind_t) (call_frame_t *frame,
-                                         int op_ret, int op_errno, dict_t *dict);
+                                          int op_ret, int op_errno,
+                                          dict_t *dict, dict_t *xdata);
 
 
 struct volume_mark {
@@ -56,6 +49,69 @@ struct volume_mark {
         uint32_t usec;
 }__attribute__ ((__packed__));
 
+
+/*
+ * The enumerated type here
+ * is used to index two kind
+ * of integer arrays:
+ * - gauges
+ * - counters
+
+ * A counter is used internally,
+ * in getxattr callbacks, to count
+ * the results, categorized as
+ * the enum names suggest. So values
+ * in the counter are always non-negative.
+
+ * Gauges are part of the API.
+ * The caller passes one to the
+ * top-level aggregator function,
+ * cluster_getmarkerattr(). The gauge
+ * defines an evaluation policy for the
+ * counter. That is, at the
+ * end of the aggregation process
+ * the gauge is matched against the
+ * counter, and the policy
+ * represented by the gauge decides
+ * whether to return with success or failure,
+ * and in latter case, what particular failure
+ * case (errno).
+
+ * The rules are the following: for some index i,
+ * - if gauge[i] == 0, no requirement is set
+ *   against counter[i];
+ * - if gauge[i] > 0, counter[i] >= gauge[i]
+ *   is required;
+ * - if gauge[i] < 0, counter[i] < |gauge[i]|
+ *   is required.
+
+ * If the requirement is not met, then i is mapped
+ * to the respective errno (MCNT_ENOENT -> ENOENT),
+ * or in lack of that, EINVAL.
+
+ * Cf. evaluate_marker_results() and marker_idx_errno_map[]
+ * in libxlator.c
+
+ * We provide two default gauges, one inteded for xtime
+ * aggregation, other for volume mark aggregation. The
+ * policies they represent agree with the hard-coded
+ * one prior to gauges. Cf. marker_xtime_default_gauge
+ * and marker_uuid_default_gauge in libxlator.c
+ */
+
+typedef enum {
+        MCNT_FOUND,
+        MCNT_NOTFOUND,
+        MCNT_ENODATA,
+        MCNT_ENOTCONN,
+        MCNT_ENOENT,
+        MCNT_EOTHER,
+        MCNT_MAX
+} marker_result_idx_t;
+
+extern int marker_xtime_default_gauge[];
+extern int marker_uuid_default_gauge[];
+
 struct marker_str {
         struct volume_mark    *volmark;
         data_t                *data;
@@ -63,13 +119,8 @@ struct marker_str {
         uint32_t               host_timebuf[2];
         uint32_t               net_timebuf[2];
         int32_t                call_count;
-        unsigned               has_xtime:1;
-        int32_t                enoent_count;
-        int32_t                enotconn_count;
-        int32_t                enodata_count;
-        int32_t                noxtime_count;
-
-        int                    esomerr;
+        int                    gauge[MCNT_MAX];
+        int                    count[MCNT_MAX];
 
         xlator_specf_unwind_t  xl_specf_unwind;
         void                  *xl_local;
@@ -79,34 +130,28 @@ struct marker_str {
 
 typedef struct marker_str xl_marker_local_t;
 
-static inline gf_boolean_t
-marker_has_volinfo (xl_marker_local_t *marker)
-{
-       if (marker->volmark)
-                return _gf_true;
-       else
-                return _gf_false;
-}
-
 int32_t
 cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict);
+                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
 
 int32_t
 cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict);
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
 
 int32_t
 cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
                        const char *name, void *xl_local,
                        xlator_specf_unwind_t xl_specf_getxattr_unwind,
                        xlator_t **sub_volumes, int count, int type,
-                       char *vol_uuid);
+                       int *gauge, char *vol_uuid);
 
 int
 match_uuid_local (const char *name, char *uuid);
 
+int
+gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value);
 
-
+int
+gf_get_max_stime (xlator_t *this, dict_t *dst, char *key, data_t *value);
 
 #endif /* !_LIBXLATOR_H */
diff --git a/xlators/meta/src/Makefile.am b/xlators/meta/src/Makefile.am
index 385ff553f..f8fa7d4cb 100644
--- a/xlators/meta/src/Makefile.am
+++ b/xlators/meta/src/Makefile.am
@@ -4,7 +4,8 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/
 meta_so_SOURCES = meta.c tree.c misc.c view.c
 noinst_HEADERS = meta.h tree.h misc.h view.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall
 
 CLEANFILES = 
diff --git a/xlators/meta/src/meta-mem-types.h b/xlators/meta/src/meta-mem-types.h
index 9585b7838..62028b246 100644
--- a/xlators/meta/src/meta-mem-types.h
+++ b/xlators/meta/src/meta-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __META_MEM_TYPES_H__
 #define __META_MEM_TYPES_H__
 
diff --git a/xlators/meta/src/meta.c b/xlators/meta/src/meta.c
index 412b4a2b5..e69719f3c 100644
--- a/xlators/meta/src/meta.c
+++ b/xlators/meta/src/meta.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
diff --git a/xlators/meta/src/meta.h b/xlators/meta/src/meta.h
index 5636487c9..73e0e50db 100644
--- a/xlators/meta/src/meta.h
+++ b/xlators/meta/src/meta.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __META_H__
 #define __META_H__
 
diff --git a/xlators/meta/src/misc.c b/xlators/meta/src/misc.c
index bea07e70c..1a8dfa806 100644
--- a/xlators/meta/src/misc.c
+++ b/xlators/meta/src/misc.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <unistd.h>
 #include <sys/uio.h>
 
diff --git a/xlators/meta/src/misc.h b/xlators/meta/src/misc.h
index f934a6d8d..30dd10e34 100644
--- a/xlators/meta/src/misc.h
+++ b/xlators/meta/src/misc.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MISC_H__
 #define __MISC_H__
 
diff --git a/xlators/meta/src/tree.c b/xlators/meta/src/tree.c
index cad5cbd71..dacbd665a 100644
--- a/xlators/meta/src/tree.c
+++ b/xlators/meta/src/tree.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -123,7 +113,7 @@ lookup_meta_entry (meta_dirent_t *root, const char *path,
 	    gf_asprintf (remain, "/%s/%s", *remain, piece);
 	  else
 	    gf_asprintf (remain, "/%s", piece);
-	  if (tmp) GF_FREE (tmp);
+	  GF_FREE (tmp);
 	  piece = strtok (NULL, "/");
 	}
       }
diff --git a/xlators/meta/src/tree.h b/xlators/meta/src/tree.h
index 8157148db..985df3bd7 100644
--- a/xlators/meta/src/tree.h
+++ b/xlators/meta/src/tree.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __TREE_H__
 #define __TREE_H__
 
diff --git a/xlators/meta/src/view.c b/xlators/meta/src/view.c
index ba3c30e0b..b4e2d64a2 100644
--- a/xlators/meta/src/view.c
+++ b/xlators/meta/src/view.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/meta/src/view.h b/xlators/meta/src/view.h
index 440c0d34d..2eff6126e 100644
--- a/xlators/meta/src/view.h
+++ b/xlators/meta/src/view.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __VIEW_H__
 #define __VIEW_H__
 
diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
index 817f02735..b109e6dff 100644
--- a/xlators/mgmt/glusterd/src/Makefile.am
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -1,27 +1,48 @@
 xlator_LTLIBRARIES = glusterd.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mgmt
-glusterd_la_CPPFLAGS = "-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\""
-glusterd_la_LDFLAGS = -module -avoidversion $(LIBXML2_LIBS)
-glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c glusterd-op-sm.c \
-	glusterd-utils.c glusterd-rpc-ops.c glusterd-store.c glusterd-handshake.c \
-	glusterd-pmap.c glusterd-volgen.c glusterd-rebalance.c glusterd-quota.c \
+glusterd_la_CPPFLAGS = $(AM_CPPFLAGS) "-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\""
+glusterd_la_LDFLAGS = -module -avoid-version
+if ENABLE_BD_XLATOR
+glusterd_la_LDFLAGS += -llvm2app
+endif
+glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \
+	glusterd-op-sm.c glusterd-utils.c glusterd-rpc-ops.c \
+	glusterd-store.c glusterd-handshake.c glusterd-pmap.c \
+	glusterd-volgen.c glusterd-rebalance.c glusterd-quota.c \
 	glusterd-geo-rep.c glusterd-replace-brick.c glusterd-log-ops.c \
-	glusterd-volume-ops.c glusterd-brick-ops.c glusterd-mountbroker.c
+	glusterd-volume-ops.c glusterd-brick-ops.c glusterd-mountbroker.c \
+	glusterd-syncop.c glusterd-hooks.c glusterd-volume-set.c \
+        glusterd-locks.c
 
 glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 		     $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		     $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+		     $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		     $(XML_LIBS) -lcrypto
 
-noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h glusterd-sm.h \
-	glusterd-store.h glusterd-mem-types.h glusterd-pmap.h glusterd-volgen.h \
-        glusterd-mountbroker.h
+noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \
+	glusterd-sm.h glusterd-store.h glusterd-mem-types.h \
+	glusterd-pmap.h glusterd-volgen.h glusterd-mountbroker.h \
+	glusterd-syncop.h glusterd-hooks.h glusterd-locks.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)\
-	-I$(rpclibdir) -L$(xlatordir)/ -I$(CONTRIBDIR)/rbtree -I$(top_srcdir)/rpc/xdr/src\
-	-I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/uuid -I$(top_srcdir)/contrib/md5  -DSBIN_DIR=\"$(sbindir)\"\
-        -DDATADIR=\"$(localstatedir)\" -DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
-        -DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(LIBXML2_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(rpclibdir) -I$(CONTRIBDIR)/rbtree \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(CONTRIBDIR)/uuid \
+	-DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \
+	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
+        -DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS)
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+AM_LDFLAGS = -L$(xlatordir)
 
 CLEANFILES =
+
+install-data-hook:
+
+if GF_INSTALL_VAR_LIB_GLUSTERD
+	$(mkdir_p) $(localstatedir)/lib/
+	(stat $(sysconfdir)/glusterd && \
+	    mv $(sysconfdir)/glusterd $(localstatedir)/lib/) || true;
+	(ln -sf $(localstatedir)/lib/glusterd $(sysconfdir)/glusterd) || true;
+endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index fc9c9cf0a..1804dd02e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -133,6 +123,15 @@ gd_addbr_validate_stripe_count (glusterd_volinfo_t *volinfo, int stripe_count,
                                 volinfo->volname);
                         ret = 0;
                         goto out;
+                } else {
+                        snprintf (err_str, err_len, "Incorrect number of "
+                                  "bricks (%d) supplied for changing volume's "
+                                  "stripe count to %d, need at least %d bricks",
+                                  (total_bricks - volinfo->brick_count),
+                                  stripe_count,
+                                  (volinfo->replica_count * stripe_count));
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
                 }
                 break;
         case GF_CLUSTER_TYPE_STRIPE:
@@ -214,6 +213,16 @@ gd_addbr_validate_replica_count (glusterd_volinfo_t *volinfo, int replica_count,
                                 volinfo->volname);
                         ret = 0;
                         goto out;
+                } else {
+                        snprintf (err_str, err_len, "Incorrect number of "
+                                  "bricks (%d) supplied for changing volume's "
+                                  "replica count to %d, need at least %d "
+                                  "bricks",
+                                  (total_bricks - volinfo->brick_count),
+                                  replica_count, (volinfo->dist_leaf_count *
+                                                  replica_count));
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
                 }
                 break;
         case GF_CLUSTER_TYPE_REPLICATE:
@@ -256,8 +265,10 @@ out:
 }
 
 static int
-gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo, int32_t replica_count,
-                                int32_t brick_count, char *err_str)
+gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo,
+                                int32_t replica_count,
+                                int32_t brick_count, char *err_str,
+                                size_t err_len)
 {
         int ret = -1;
         int replica_nodes = 0;
@@ -265,7 +276,7 @@ gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo, int32_t replica_cou
         switch (volinfo->type) {
         case GF_CLUSTER_TYPE_NONE:
         case GF_CLUSTER_TYPE_STRIPE:
-                snprintf (err_str, 2048,
+                snprintf (err_str, err_len,
                           "replica count (%d) option given for non replicate "
                           "volume %s", replica_count, volinfo->volname);
                 gf_log (THIS->name, GF_LOG_WARNING, "%s", err_str);
@@ -275,7 +286,7 @@ gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo, int32_t replica_cou
         case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
                 /* in remove brick, you can only reduce the replica count */
                 if (replica_count > volinfo->replica_count) {
-                        snprintf (err_str, 2048,
+                        snprintf (err_str, err_len,
                                   "given replica count (%d) option is more "
                                   "than volume %s's replica count (%d)",
                                   replica_count, volinfo->volname,
@@ -284,15 +295,30 @@ gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo, int32_t replica_cou
                         goto out;
                 }
                 if (replica_count == volinfo->replica_count) {
+                        /* This means the 'replica N' option on CLI was
+                           redundant. Check if the total number of bricks given
+                           for removal is same as 'dist_leaf_count' */
+                        if (brick_count % volinfo->dist_leaf_count) {
+                                snprintf (err_str, err_len,
+                                          "number of bricks provided (%d) is "
+                                          "not valid. need at least %d "
+                                          "(or %dxN)", brick_count,
+                                          volinfo->dist_leaf_count,
+                                          volinfo->dist_leaf_count);
+                                gf_log (THIS->name, GF_LOG_WARNING, "%s",
+                                        err_str);
+                                goto out;
+                        }
                         ret = 1;
                         goto out;
                 }
 
-                replica_nodes = ((volinfo->brick_count / volinfo->replica_count) *
+                replica_nodes = ((volinfo->brick_count /
+                                  volinfo->replica_count) *
                                  (volinfo->replica_count - replica_count));
 
                 if (brick_count % replica_nodes) {
-                        snprintf (err_str, 2048,
+                        snprintf (err_str, err_len,
                                   "need %d(xN) bricks for reducing replica "
                                   "count of the volume from %d to %d",
                                   replica_nodes, volinfo->replica_count,
@@ -307,11 +333,9 @@ out:
         return ret;
 }
 
-
-
 /* Handler functions */
 int
-glusterd_handle_add_brick (rpcsvc_request_t *req)
+__glusterd_handle_add_brick (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -319,7 +343,6 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
         char                            *bricks = NULL;
         char                            *volname = NULL;
         int                             brick_count = 0;
-        char                            *brick_list = NULL;
         void                            *cli_rsp = NULL;
         char                            err_str[2048] = {0,};
         gf_cli_rsp                      rsp = {0,};
@@ -335,15 +358,16 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 snprintf (err_str, sizeof (err_str), "Garbage args received");
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received add brick req");
+        gf_log (this->name, GF_LOG_INFO, "Received add brick req");
 
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
@@ -353,59 +377,62 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
                         snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the buffer");
+                                  "the command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
 
-        gf_cmd_log ("Volume add-brick", "on volname: %s attempted",
-                    volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
                 snprintf (err_str, sizeof (err_str), "Unable to get volume "
                           "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         if (!(ret = glusterd_check_volume_exists (volname))) {
                 ret = -1;
-                snprintf(err_str, 2048, "Volume %s does not exist", volname);
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                snprintf (err_str, sizeof (err_str), "Volume %s does not exist",
+                          volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
                 snprintf (err_str, sizeof (err_str), "Unable to get volume "
                           "brick count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "replica-count", &replica_count);
         if (!ret) {
-                gf_log (THIS->name, GF_LOG_INFO, "replica-count is %d",
+                gf_log (this->name, GF_LOG_INFO, "replica-count is %d",
                         replica_count);
         }
 
         ret = dict_get_int32 (dict, "stripe-count", &stripe_count);
         if (!ret) {
-                gf_log (THIS->name, GF_LOG_INFO, "stripe-count is %d",
+                gf_log (this->name, GF_LOG_INFO, "stripe-count is %d",
                         stripe_count);
         }
 
+        if (!dict_get (dict, "force")) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get flag");
+                goto out;
+        }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
                 snprintf (err_str, sizeof (err_str), "Unable to get volinfo "
                           "for volume name %s", volname);
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
 
         }
@@ -421,10 +448,10 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
                         goto brick_val;
 
                 if ((brick_count % volinfo->dist_leaf_count) != 0) {
-                        snprintf(err_str, 2048, "Incorrect number of bricks"
-                                 " supplied %d with count %d",
+                        snprintf (err_str, sizeof (err_str), "Incorrect number "
+                                  "of bricks supplied %d with count %d",
                                  brick_count, volinfo->dist_leaf_count);
-                        gf_log("glusterd", GF_LOG_ERROR, "%s", err_str);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         ret = -1;
                         goto out;
                 }
@@ -440,7 +467,7 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
                                                       err_str,
                                                       sizeof (err_str));
                 if (ret == -1) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         goto out;
                 }
 
@@ -450,7 +477,7 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
 
                 ret = dict_set_int32 (dict, "stripe-count", stripe_count);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to set the stripe-count in dict");
                         goto out;
                 }
@@ -462,7 +489,7 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
                                                &type, err_str,
                                                sizeof (err_str));
         if (ret == -1) {
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
@@ -472,7 +499,7 @@ glusterd_handle_add_brick (rpcsvc_request_t *req)
 
         ret = dict_set_int32 (dict, "replica-count", replica_count);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "failed to set the replica-count in dict");
                 goto out;
         }
@@ -482,51 +509,123 @@ brick_val:
         if (ret) {
                 snprintf (err_str, sizeof (err_str), "Unable to get volume "
                           "bricks");
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_cmd_log ("Volume add-brick", "volname: %s type %d count:%d bricks:%s"
-                    ,volname, volinfo->type, brick_count, brick_list);
-
         if (type != volinfo->type) {
                 ret = dict_set_int32 (dict, "type", type);
                 if (ret)
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to set the new type in dict");
         }
 
-        ret = glusterd_op_begin (req, GD_OP_ADD_BRICK, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
 
 out:
-        gf_cmd_log ("Volume add-brick","on volname: %s %s", volname,
-                   (ret != 0)? "FAILED" : "SUCCESS");
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
                 rsp.op_ret = -1;
                 rsp.op_errno = 0;
                 if (err_str[0] == '\0')
                         snprintf (err_str, sizeof (err_str), "Operation failed");
                 rsp.op_errstr = err_str;
                 cli_rsp = &rsp;
-                glusterd_submit_reply(req, cli_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gf_cli_rsp);
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
                 ret = 0; //sent error to cli, prevent second reply
         }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        free (cli_req.dict.dict_val); //its malloced by xdr
+
+        return ret;
+}
+
+int
+glusterd_handle_add_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_add_brick);
+}
+
+static int
+subvol_matcher_init (int **subvols, int count)
+{
+        int ret = -1;
 
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
+        *subvols = GF_CALLOC (count, sizeof(int), gf_gld_mt_int);
+        if (*subvols)
+                ret = 0;
 
         return ret;
 }
 
+static void
+subvol_matcher_update (int *subvols, glusterd_volinfo_t *volinfo,
+                       glusterd_brickinfo_t *brickinfo)
+{
+        glusterd_brickinfo_t *tmp        = NULL;
+        int32_t               sub_volume = 0;
+        int                   pos        = 0;
+
+        list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
+
+                if (strcmp (tmp->hostname, brickinfo->hostname) ||
+                    strcmp (tmp->path, brickinfo->path)) {
+                        pos++;
+                        continue;
+                }
+                gf_log (THIS->name, GF_LOG_DEBUG, LOGSTR_FOUND_BRICK,
+                                        brickinfo->hostname, brickinfo->path,
+                                        volinfo->volname);
+                sub_volume = (pos / volinfo->dist_leaf_count);
+                subvols[sub_volume]++;
+                break;
+        }
+
+}
+
+static int
+subvol_matcher_verify (int *subvols, glusterd_volinfo_t *volinfo, char *err_str,
+                       size_t err_len, char *vol_type, int replica_count)
+{
+        int i = 0;
+        int ret = 0;
+        int count = volinfo->replica_count-replica_count;
+
+        if (replica_count) {
+                for (i = 0; i < volinfo->subvol_count; i++) {
+                        if (subvols[i] != count) {
+                                ret = -1;
+                                snprintf (err_str, err_len, "Remove exactly %d"
+                                " brick(s) from each subvolume.", count);
+                                break;
+                        }
+                }
+                return ret;
+        }
+
+       do {
+
+                if (subvols[i] % volinfo->dist_leaf_count == 0) {
+                        continue;
+                } else {
+                        ret = -1;
+                        snprintf (err_str, err_len,
+                                "Bricks not from same subvol for %s", vol_type);
+                        break;
+                }
+        } while (++i < volinfo->subvol_count);
+
+        return ret;
+}
+
+static void
+subvol_matcher_destroy (int *subvols)
+{
+        GF_FREE (subvols);
+}
 
 int
-glusterd_handle_remove_brick (rpcsvc_request_t *req)
+__glusterd_handle_remove_brick (rpcsvc_request_t *req)
 {
         int32_t                   ret              = -1;
         gf_cli_req                cli_req          = {{0,}};
@@ -538,33 +637,30 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
         int                       i                = 1;
         glusterd_volinfo_t       *volinfo          = NULL;
         glusterd_brickinfo_t     *brickinfo        = NULL;
-        int32_t                   pos              = 0;
-        int32_t                   sub_volume       = 0;
-        int32_t                   sub_volume_start = 0;
-        int32_t                   sub_volume_end   = 0;
-        glusterd_brickinfo_t     *tmp              = NULL;
+        int                      *subvols          = NULL;
         char                      err_str[2048]    = {0};
         gf_cli_rsp                rsp              = {0,};
         void                     *cli_rsp          = NULL;
         char                      vol_type[256]    = {0,};
         int32_t                   replica_count    = 0;
-        int32_t                   brick_index      = 0;
-        int32_t                   tmp_brick_idx    = 0;
-        int                       found            = 0;
-        int                       diff_count       = 0;
         char                     *volname          = 0;
+        xlator_t                 *this             = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
+                snprintf (err_str, sizeof (err_str), "Received garbage args");
                 goto out;
         }
 
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received rem brick req");
+        gf_log (this->name, GF_LOG_INFO, "Received rem brick req");
 
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
@@ -574,52 +670,61 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get volname");
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_cmd_log ("Volume remove-brick","on volname: %s attempted", volname);
         ret = dict_get_int32 (dict, "count", &count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                snprintf (err_str, sizeof (err_str), "Unable to get brick "
+                          "count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                 snprintf (err_str, 2048, "Volume %s does not exist",
-                          volname);
-                 gf_log ("", GF_LOG_ERROR, "%s", err_str);
+                 snprintf (err_str, sizeof (err_str),"Volume %s does not exist",
+                           volname);
+                 gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                  goto out;
         }
 
         ret = dict_get_int32 (dict, "replica-count", &replica_count);
         if (!ret) {
-                gf_log (THIS->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_INFO,
                         "request to change replica-count to %d", replica_count);
                 ret = gd_rmbr_validate_replica_count (volinfo, replica_count,
-                                                      count, err_str);
+                                                      count, err_str,
+                                                      sizeof (err_str));
                 if (ret < 0) {
-                        /* logging and error msg are done in above function itself */
+                        /* logging and error msg are done in above function
+                           itself */
                         goto out;
                 }
                 dict_del (dict, "replica-count");
                 if (ret) {
                         replica_count = 0;
                 } else {
-                        ret = dict_set_int32 (dict, "replica-count", replica_count);
+                        ret = dict_set_int32 (dict, "replica-count",
+                                              replica_count);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to set the replica_count in dict");
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to set the replica_count "
+                                        "in dict");
                                 goto out;
                         }
                 }
@@ -636,41 +741,54 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                 strcpy (vol_type, "distribute");
         }
 
-	/* Do not allow remove-brick if the volume is plain stripe */
+	/* Do not allow remove-brick if the volume is a stripe volume*/
 	if ((volinfo->type == GF_CLUSTER_TYPE_STRIPE) &&
             (volinfo->brick_count == volinfo->stripe_count)) {
-                snprintf (err_str, 2048, "Removing brick from a plain stripe is not allowed");
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                snprintf (err_str, sizeof (err_str),
+                          "Removing brick from a stripe volume is not allowed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 ret = -1;
                 goto out;
 	}
 
-	/* Do not allow remove-brick if the bricks given is less than the replica count
-	   or stripe count */
-        if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE) &&
-            !(volinfo->brick_count <= volinfo->dist_leaf_count)) {
-                if (volinfo->dist_leaf_count &&
-                    (count % volinfo->dist_leaf_count)) {
-                        snprintf (err_str, 2048, "Remove brick incorrect"
-                                  " brick count of %d for %s %d",
-                                  count, vol_type, volinfo->dist_leaf_count);
-                        gf_log ("", GF_LOG_ERROR, "%s", err_str);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
 	if (!replica_count &&
             (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) &&
             (volinfo->brick_count == volinfo->dist_leaf_count)) {
-                snprintf (err_str, 2048, "Removing bricks from stripe-replicate"
+                snprintf (err_str, sizeof(err_str),
+                          "Removing bricks from stripe-replicate"
                           " configuration is not allowed without reducing "
                           "replica or stripe count explicitly.");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+	if (!replica_count &&
+            (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+            (volinfo->brick_count == volinfo->dist_leaf_count)) {
+                snprintf (err_str, sizeof (err_str),
+                          "Removing bricks from replicate configuration "
+                          "is not allowed without reducing replica count "
+                          "explicitly.");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 ret = -1;
                 goto out;
         }
 
+	/* Do not allow remove-brick if the bricks given is less than
+           the replica count or stripe count */
+        if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE)) {
+                if (volinfo->dist_leaf_count &&
+                    (count % volinfo->dist_leaf_count)) {
+                        snprintf (err_str, sizeof (err_str), "Remove brick "
+                                  "incorrect brick count of %d for %s %d",
+                                  count, vol_type, volinfo->dist_leaf_count);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
         brick_list = GF_MALLOC (120000 * sizeof(*brick_list),gf_common_mt_char);
 
         if (!brick_list) {
@@ -679,23 +797,32 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
         }
 
         strcpy (brick_list, " ");
+
+        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+            (volinfo->subvol_count > 1)) {
+                ret = subvol_matcher_init (&subvols, volinfo->subvol_count);
+                if (ret)
+                        goto out;
+        }
+
         while ( i <= count) {
-                snprintf (key, 256, "brick%d", i);
+                snprintf (key, sizeof (key), "brick%d", i);
                 ret = dict_get_str (dict, key, &brick);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get %s", key);
+                        snprintf (err_str, sizeof (err_str), "Unable to get %s",
+                                  key);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         goto out;
                 }
-                gf_log ("", GF_LOG_DEBUG, "Remove brick count %d brick: %s",
-                        i, brick);
+                gf_log (this->name, GF_LOG_DEBUG, "Remove brick count %d brick:"
+                        " %s", i, brick);
 
                 ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo,
-                                                             &brickinfo,
-                                                             GF_PATH_COMPLETE);
+                                                             &brickinfo);
                 if (ret) {
-                        snprintf(err_str, 2048,"Incorrect brick %s for volume"
-                                " %s", brick, volname);
-                        gf_log ("", GF_LOG_ERROR, "%s", err_str);
+                        snprintf (err_str, sizeof (err_str), "Incorrect brick "
+                                  "%s for volume %s", brick, volname);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         goto out;
                 }
                 strcat(brick_list, brick);
@@ -706,110 +833,146 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
                     (volinfo->brick_count <= volinfo->dist_leaf_count))
                         continue;
 
-                if (replica_count) {
-                        /* do the validation of bricks here */
-                        /* -2 because i++ is already done, and i starts with 1,
-                           instead of 0 */
-                        diff_count = (volinfo->replica_count - replica_count);
-                        brick_index = (((i -2) / diff_count) * volinfo->replica_count);
-                        tmp_brick_idx = 0;
-                        found = 0;
-                        list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-                                tmp_brick_idx++;
-                                gf_log (THIS->name, GF_LOG_TRACE,
-                                        "validate brick %s:%s (%d %d %d)",
-                                        tmp->hostname, tmp->path, tmp_brick_idx,
-                                        brick_index, volinfo->replica_count);
-                                if (tmp_brick_idx <= brick_index)
-                                        continue;
-                                if (tmp_brick_idx >
-                                    (brick_index + volinfo->replica_count))
-                                        break;
-                                if ((!strcmp (tmp->hostname,brickinfo->hostname)) &&
-                                    !strcmp (tmp->path, brickinfo->path)) {
-                                        found = 1;
-                                        break;
-                                }
-                        }
-                        if (found)
-                                continue;
+                /* Find which subvolume the brick belongs to */
+                subvol_matcher_update (subvols, volinfo, brickinfo);
+        }
 
-                        snprintf(err_str, 2048,"Bricks are from same subvol");
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "failed to validate brick %s:%s (%d %d %d)",
-                                tmp->hostname, tmp->path, tmp_brick_idx,
-                                brick_index, volinfo->replica_count);
-                        ret = -1;
-                        /* brick order is not valid */
+        /* Check if the bricks belong to the same subvolumes.*/
+        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+            (volinfo->subvol_count > 1)) {
+                ret = subvol_matcher_verify (subvols, volinfo,
+                                             err_str, sizeof(err_str),
+                                             vol_type, replica_count);
+                if (ret)
                         goto out;
-                }
-
-                pos = 0;
-                list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-
-                        if (strcmp (tmp->hostname,brickinfo->hostname) ||
-                            strcmp (tmp->path, brickinfo->path)) {
-                                pos++;
-                                continue;
-                        }
-
-                        gf_log ("", GF_LOG_INFO, "Found brick");
-                        if (!sub_volume && (volinfo->dist_leaf_count > 1)) {
-                                sub_volume = (pos / volinfo->dist_leaf_count) + 1;
-                                sub_volume_start = (volinfo->dist_leaf_count *
-                                                    (sub_volume - 1));
-                                sub_volume_end = (volinfo->dist_leaf_count *
-                                                  sub_volume) - 1;
-                        } else {
-                                if (pos < sub_volume_start ||
-                                    pos >sub_volume_end) {
-                                        ret = -1;
-                                        snprintf(err_str, 2048,"Bricks not from"
-                                                 " same subvol for %s",
-                                                 vol_type);
-                                        gf_log ("", GF_LOG_ERROR,
-                                                "%s", err_str);
-                                        goto out;
-                                }
-                        }
-                        break;
-                }
         }
-        gf_cmd_log ("Volume remove-brick","volname: %s count:%d bricks:%s",
-                    volname, count, brick_list);
 
-        ret = glusterd_op_begin (req, GD_OP_REMOVE_BRICK, dict);
-        gf_cmd_log ("Volume remove-brick","on volname: %s %s", volname,
-                    (ret) ? "FAILED" : "SUCCESS");
+        ret = glusterd_op_begin_synctask (req, GD_OP_REMOVE_BRICK, dict);
 
 out:
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
                 rsp.op_ret = -1;
                 rsp.op_errno = 0;
                 if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str), "Operation failed");
-                gf_log ("", GF_LOG_ERROR, "%s", err_str);
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 rsp.op_errstr = err_str;
                 cli_rsp = &rsp;
-                glusterd_submit_reply(req, cli_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gf_cli_rsp);
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
 
                 ret = 0; //sent error to cli, prevent second reply
 
         }
-        if (brick_list)
-                GF_FREE (brick_list);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        GF_FREE (brick_list);
+        subvol_matcher_destroy (subvols);
+        free (cli_req.dict.dict_val); //its malloced by xdr
 
         return ret;
 }
 
+int
+glusterd_handle_remove_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_remove_brick);
+}
+
+static int
+_glusterd_restart_gsync_session (dict_t *this, char *key,
+                                 data_t *value, void *data)
+{
+        char                          *slave      = NULL;
+        char                          *slave_buf  = NULL;
+        char                          *path_list  = NULL;
+        char                          *slave_vol  = NULL;
+        char                          *slave_ip   = NULL;
+        char                          *conf_path  = NULL;
+        char                         **errmsg     = NULL;
+        int                            ret        = -1;
+        glusterd_gsync_status_temp_t  *param      = NULL;
+        gf_boolean_t                   is_running = _gf_false;
+
+        param = (glusterd_gsync_status_temp_t *)data;
+
+        GF_ASSERT (param);
+        GF_ASSERT (param->volinfo);
+
+        slave = strchr(value->data, ':');
+        if (slave) {
+                slave++;
+                slave_buf = gf_strdup (slave);
+                if (!slave_buf) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
+                        goto out;
+                }
+        }
+        else
+                return 0;
+
+        ret = dict_set_dynstr (param->rsp_dict, "slave", slave_buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave");
+                if (slave_buf)
+                        GF_FREE(slave_buf);
+                goto out;
+        }
+
+        ret = glusterd_get_slave_details_confpath (param->volinfo,
+                                                   param->rsp_dict,
+                                                   &slave_ip, &slave_vol,
+                                                   &conf_path, errmsg);
+        if (ret) {
+                if (*errmsg)
+                        gf_log ("", GF_LOG_ERROR, "%s", *errmsg);
+                else
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to fetch slave or confpath details.");
+                goto out;
+        }
+
+        /* In cases that gsyncd is not running, we will not invoke it
+         * because of add-brick. */
+        ret = glusterd_check_gsync_running_local (param->volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "gsync running validation failed.");
+                goto out;
+        }
+        if (_gf_false == is_running) {
+                gf_log ("", GF_LOG_DEBUG, "gsync session for %s and %s is"
+                        " not running on this node. Hence not restarting.",
+                        param->volinfo->volname, slave);
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_get_local_brickpaths (param->volinfo, &path_list);
+        if (!path_list) {
+                gf_log ("", GF_LOG_DEBUG, "This node not being part of"
+                        " volume should not be running gsyncd. Hence"
+                        " no gsyncd process to restart.");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_check_restart_gsync_session (param->volinfo, slave,
+                                                    param->rsp_dict, path_list,
+                                                    conf_path, 0);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to restart gsync session.");
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
+        return ret;
+}
 
 /* op-sm */
 
@@ -817,17 +980,22 @@ int
 glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                                 char  *bricks, dict_t *dict)
 {
-        glusterd_brickinfo_t *brickinfo     = NULL;
-        char                 *brick         = NULL;
-        int32_t               i             = 1;
-        char                 *brick_list    = NULL;
-        char                 *free_ptr1     = NULL;
-        char                 *free_ptr2     = NULL;
-        char                 *saveptr       = NULL;
-        int32_t               ret           = -1;
-        int32_t               stripe_count  = 0;
-        int32_t               replica_count = 0;
-        int32_t               type          = 0;
+        char                         *brick          = NULL;
+        int32_t                       i              = 1;
+        char                         *brick_list     = NULL;
+        char                         *free_ptr1      = NULL;
+        char                         *free_ptr2      = NULL;
+        char                         *saveptr        = NULL;
+        int32_t                       ret            = -1;
+        int32_t                       stripe_count   = 0;
+        int32_t                       replica_count  = 0;
+        int32_t                       type           = 0;
+        glusterd_brickinfo_t         *brickinfo      = NULL;
+        glusterd_gsync_status_temp_t  param          = {0, };
+        gf_boolean_t                  restart_needed = 0;
+        char                          msg[1024] __attribute__((unused)) = {0, };
+        int                           caps           = 0;
+        int                           brickid        = 0;
 
         GF_ASSERT (volinfo);
 
@@ -855,11 +1023,17 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                                 "type is set %d, need to change it", type);
         }
 
+        brickid = glusterd_get_next_available_brickid (volinfo);
+        if (brickid < 0)
+                goto out;
         while ( i <= count) {
-                ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
                 if (ret)
                         goto out;
 
+                GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo,
+                                                      brickid++);
+
                 ret = glusterd_resolve_brick (brickinfo);
                 if (ret)
                         goto out;
@@ -885,46 +1059,94 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         if (stripe_count) {
                 volinfo->stripe_count = stripe_count;
         }
-        volinfo->dist_leaf_count = (volinfo->stripe_count *
-                                    volinfo->replica_count);
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
 
         /* backward compatibility */
         volinfo->sub_count = ((volinfo->dist_leaf_count == 1) ? 0:
                               volinfo->dist_leaf_count);
 
+        volinfo->subvol_count = (volinfo->brick_count /
+                                 volinfo->dist_leaf_count);
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret)
+                goto out;
+
+        ret = 0;
+        if (GLUSTERD_STATUS_STARTED != volinfo->status)
+                goto out;
+
         brick_list = gf_strdup (bricks);
         free_ptr2 = brick_list;
         i = 1;
 
         if (count)
                 brick = strtok_r (brick_list+1, " \n", &saveptr);
-
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret)
-                goto out;
+#ifdef HAVE_BD_XLATOR
+        if (brickinfo->vg[0])
+                caps = CAPS_BD | CAPS_THIN |
+                        CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+#endif
 
         while (i <= count) {
-
                 ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_PARTIAL);
+                                                              &brickinfo);
                 if (ret)
                         goto out;
+#ifdef HAVE_BD_XLATOR
+                /* Check for VG/thin pool if its BD volume */
+                if (brickinfo->vg[0]) {
+                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_CRITICAL, "%s", msg);
+                                goto out;
+                        }
+                        /* if anyone of the brick does not have thin support,
+                           disable it for entire volume */
+                        caps &= brickinfo->caps;
+                } else
+                        caps = 0;
+#endif
 
-                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                        ret = glusterd_brick_start (volinfo, brickinfo);
-                        if (ret)
+                if (uuid_is_null (brickinfo->uuid)) {
+                        ret = glusterd_resolve_brick (brickinfo);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                        brickinfo->hostname, brickinfo->path);
                                 goto out;
+                        }
                 }
+
+                ret = glusterd_brick_start (volinfo, brickinfo,
+                                            _gf_true);
+                if (ret)
+                        goto out;
                 i++;
                 brick = strtok_r (NULL, " \n", &saveptr);
+
+                /* Check if the brick is added in this node, and set
+                 * the restart_needed flag. */
+                if ((!uuid_compare (brickinfo->uuid, MY_UUID)) &&
+                    !restart_needed) {
+                        restart_needed = 1;
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Restart gsyncd session, if it's already "
+                                "running.");
+                }
         }
 
+        /* If the restart_needed flag is set, restart gsyncd sessions for that
+         * particular master with all the slaves. */
+        if (restart_needed) {
+                param.rsp_dict = dict;
+                param.volinfo = volinfo;
+                dict_foreach (volinfo->gsync_slaves,
+                              _glusterd_restart_gsync_session, &param);
+        }
+        volinfo->caps = caps;
 out:
-        if (free_ptr1)
-                GF_FREE (free_ptr1);
-        if (free_ptr2)
-                GF_FREE (free_ptr2);
+        GF_FREE (free_ptr1);
+        GF_FREE (free_ptr2);
 
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -936,7 +1158,6 @@ glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
                                   int force, int *need_migrate)
 {
         glusterd_brickinfo_t    *brickinfo = NULL;
-        char                    *dup_brick = NULL;
         int32_t                  ret = -1;
         glusterd_conf_t         *priv = NULL;
 
@@ -946,12 +1167,8 @@ glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
         priv = THIS->private;
         GF_ASSERT (priv);
 
-        dup_brick = gf_strdup (brick);
-        if (!dup_brick)
-                goto out;
-
-        ret = glusterd_volume_brickinfo_get_by_brick (dup_brick, volinfo,
-                                                      &brickinfo, GF_PATH_COMPLETE);
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                      &brickinfo);
         if (ret)
                 goto out;
 
@@ -959,30 +1176,27 @@ glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
         if (ret)
                 goto out;
 
-        if (!uuid_compare (brickinfo->uuid, priv->uuid)) {
+        glusterd_volinfo_reset_defrag_stats (volinfo);
+
+        if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
                 /* Only if the brick is in this glusterd, do the rebalance */
                 if (need_migrate)
                         *need_migrate = 1;
         }
 
         if (force) {
-                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                        ret = glusterd_brick_stop (volinfo, brickinfo);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR, "Unable to stop "
-                                        "glusterfs, ret: %d", ret);
-                                goto out;
-                        }
+                ret = glusterd_brick_stop (volinfo, brickinfo,
+                                           _gf_true);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Unable to stop "
+                                "glusterfs, ret: %d", ret);
                 }
-                glusterd_delete_brick (volinfo, brickinfo);
                 goto out;
         }
 
         brickinfo->decommissioned = 1;
+        ret = 0;
 out:
-        if (dup_brick)
-                GF_FREE (dup_brick);
-
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -993,6 +1207,7 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
         int                                     ret = 0;
         char                                    *volname = NULL;
         int                                     count = 0;
+        int                                     replica_count = 0;
         int                                     i = 0;
         char                                    *bricks    = NULL;
         char                                    *brick_list = NULL;
@@ -1001,16 +1216,31 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
         char                                    *brick = NULL;
         glusterd_brickinfo_t                    *brickinfo = NULL;
         glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_conf_t                         *priv = NULL;
+        xlator_t                                *this = NULL;
         char                                    msg[2048] = {0,};
         gf_boolean_t                            brick_alloc = _gf_false;
         char                                    *all_bricks = NULL;
         char                                    *str_ret = NULL;
+        gf_boolean_t                            is_force = _gf_false;
 
-        priv = THIS->private;
-        if (!priv)
-                goto out;
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (dict, "replica-count", &replica_count);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to get replica count");
+        }
 
+        if (replica_count > 0) {
+                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
+                                        msg, sizeof(msg));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
+                        goto out;
+                }
+        }
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_ERROR,
@@ -1025,6 +1255,10 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         if (glusterd_is_rb_ongoing (volinfo)) {
                 snprintf (msg, sizeof (msg), "Replace brick is in progress on "
                           "volume %s. Please retry after replace-brick "
@@ -1055,6 +1289,8 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
         if (bricks) {
                 brick_list = gf_strdup (bricks);
                 all_bricks = gf_strdup (bricks);
@@ -1078,24 +1314,14 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
 
                 }
 
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_PARTIAL);
-                if (!ret) {
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
+                if (ret) {
                         gf_log (THIS->name, GF_LOG_ERROR,
-                                "Adding duplicate brick: %s", brick);
-                        ret = -1;
+                                "Add-brick: Unable"
+                                " to get brickinfo");
                         goto out;
-                } else {
-                        ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Add-brick: Unable"
-                                        " to get brickinfo");
-                                goto out;
-                        }
-                        brick_alloc = _gf_true;
                 }
+                brick_alloc = _gf_true;
 
                 ret = glusterd_new_brick_validate (brick, brickinfo, msg,
                                                    sizeof (msg));
@@ -1105,11 +1331,22 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
-                if (!uuid_compare (brickinfo->uuid, priv->uuid)) {
-                        ret = glusterd_brick_create_path (brickinfo->hostname,
-                                                          brickinfo->path,
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+#ifdef HAVE_BD_XLATOR
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 1, msg);
+                                if (ret) {
+                                        gf_log (THIS->name, GF_LOG_ERROR, "%s",
+                                                msg);
+                                        *op_errstr = gf_strdup (msg);
+                                        goto out;
+                                }
+                        }
+#endif
+
+                        ret = glusterd_validate_and_create_brickpath (brickinfo,
                                                           volinfo->volume_id,
-                                                          0777, op_errstr);
+                                                          op_errstr, is_force);
                         if (ret)
                                 goto out;
                 }
@@ -1122,14 +1359,11 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
         }
 
 out:
-        if (free_ptr)
-                GF_FREE (free_ptr);
+        GF_FREE (free_ptr);
         if (brick_alloc && brickinfo)
                 glusterd_brickinfo_delete (brickinfo);
-        if (str_ret)
-                GF_FREE (str_ret);
-        if (all_bricks)
-                GF_FREE (all_bricks);
+        GF_FREE (str_ret);
+        GF_FREE (all_bricks);
 
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
@@ -1139,33 +1373,54 @@ out:
 int
 glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
 {
-        int                 ret         = -1;
-        char               *volname     = NULL;
-        glusterd_volinfo_t *volinfo     = NULL;
-        char               *errstr      = NULL;
-        int32_t             brick_count = 0;
-        char                msg[2048]   = {0,};
-        int32_t             flag        = 0;
-        gf1_op_commands     cmd         = GF_OP_CMD_NONE;
+        int                     ret         = -1;
+        char                   *volname     = NULL;
+        glusterd_volinfo_t     *volinfo     = NULL;
+        char                   *errstr      = NULL;
+        int32_t                 brick_count = 0;
+        char                    msg[2048]   = {0,};
+        int32_t                 flag        = 0;
+        gf1_op_commands         cmd         = GF_OP_CMD_NONE;
+        char                   *task_id_str = NULL;
+        xlator_t               *this        = NULL;
+        int                     i           = 1;
+        char                    key[256]    = {0,};
+        char                   *brick       = NULL;
+        glusterd_brickinfo_t   *brickinfo   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
+                                msg, sizeof(msg));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Volume %s does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "Volume %s does not exist", volname);
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         if (glusterd_is_rb_ongoing (volinfo)) {
                 snprintf (msg, sizeof (msg), "Replace brick is in progress on "
                           "volume %s. Please retry after replace-brick "
                           "operation is committed or aborted", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
@@ -1173,11 +1428,26 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
 
         ret = dict_get_int32 (dict, "command", &flag);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get brick count");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to get brick command");
                 goto out;
         }
         cmd = flag;
 
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count");
+                goto out;
+        }
+
+        ret = 0;
+        if (volinfo->brick_count == brick_count) {
+                errstr = gf_strdup ("Deleting all the bricks of the "
+                                    "volume is not allowed");
+                ret = -1;
+                goto out;
+        }
+
         ret = -1;
         switch (cmd) {
         case GF_OP_CMD_NONE:
@@ -1190,63 +1460,114 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
 
         case GF_OP_CMD_START:
         {
+                if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+                    dict_get (dict, "replica-count")) {
+                        snprintf (msg, sizeof(msg), "Migration of data is not "
+                                  "needed when reducing replica count. Use the"
+                                  " 'force' option");
+                        errstr = gf_strdup (msg);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+
                 if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                        snprintf (msg, sizeof (msg), "Volume %s needs to be started "
-                                  "before remove-brick (you can use 'force' or "
-                                  "'commit' to override this behavior)",
+                        snprintf (msg, sizeof (msg), "Volume %s needs to be "
+                                  "started before remove-brick (you can use "
+                                  "'force' or 'commit' to override this "
+                                  "behavior)", volinfo->volname);
+                        errstr = gf_strdup (msg);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+                if (!gd_is_remove_brick_committed (volinfo)) {
+                        snprintf (msg, sizeof (msg), "An earlier remove-brick "
+                                  "task exists for volume %s. Either commit it"
+                                  " or stop it before starting a new task.",
                                   volinfo->volname);
                         errstr = gf_strdup (msg);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", errstr);
+                        gf_log (this->name, GF_LOG_ERROR, "Earlier remove-brick"
+                                " task exists for volume %s.",
+                                volinfo->volname);
                         goto out;
                 }
                 if (glusterd_is_defrag_on(volinfo)) {
-                        errstr = gf_strdup("Rebalance is in progress. Please retry"
-                                           " after completion");
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", errstr);
+                        errstr = gf_strdup("Rebalance is in progress. Please "
+                                           "retry after completion");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
                         goto out;
                 }
-                break;
-        }
 
-        case GF_OP_CMD_PAUSE:
-        case GF_OP_CMD_ABORT:
-        {
-                if (!volinfo->decommission_in_progress) {
-                        errstr = gf_strdup("remove-brick is not in progress");
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", errstr);
-                        goto out;
+                if (is_origin_glusterd (dict)) {
+                        ret = glusterd_generate_and_set_task_id
+                                (dict, GF_REMOVE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
+                                goto out;
+                        }
+                } else {
+                        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Missing remove-brick-id");
+                                ret = 0;
+                        }
                 }
                 break;
         }
 
+        case GF_OP_CMD_STOP:
+                ret = 0;
+                break;
+
         case GF_OP_CMD_COMMIT:
                 if (volinfo->decommission_in_progress) {
                         errstr = gf_strdup ("use 'force' option as migration "
                                             "is in progress");
                         goto out;
                 }
+
+                /* Do not allow commit if the bricks are not decommissioned */
+                for ( i = 1; i <= brick_count; i++ ) {
+                        snprintf (key, sizeof (key), "brick%d", i);
+                        ret = dict_get_str (dict, key, &brick);
+                        if (ret) {
+                                snprintf (msg, sizeof (msg),
+                                          "Unable to get %s", key);
+                                errstr = gf_strdup (msg);
+                                goto out;
+                        }
+
+                        ret =
+                        glusterd_volume_brickinfo_get_by_brick(brick, volinfo,
+                                                               &brickinfo);
+                        if (ret) {
+                                snprintf (msg, sizeof (msg), "Incorrect brick "
+                                          "%s for volume %s", brick, volname);
+                                errstr = gf_strdup (msg);
+                                goto out;
+                        }
+                        if ( !brickinfo->decommissioned ) {
+                                snprintf (msg, sizeof (msg), "Brick %s "
+                                          "is not decommissioned. "
+                                          "Use start or force option",
+                                          brick);
+                                errstr = gf_strdup (msg);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
                 break;
 
         case GF_OP_CMD_COMMIT_FORCE:
                 break;
         }
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get brick count");
-                goto out;
-        }
-
         ret = 0;
-        if (volinfo->brick_count == brick_count) {
-                errstr = gf_strdup ("Deleting all the bricks of the "
-                                    "volume is not allowed");
-                ret = -1;
-                goto out;
-        }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         if (ret && errstr) {
                 if (op_errstr)
                         *op_errstr = errstr;
@@ -1291,6 +1612,7 @@ glusterd_remove_brick_migrate_cbk (glusterd_volinfo_t *volinfo,
                                 brickinfo->path);
                         brickinfo->decommissioned = 0;
                         if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                            /*TODO: use the 'atomic' flavour of brick_stop*/
                                 ret = glusterd_brick_stop (volinfo, brickinfo);
                                 if (ret) {
                                         gf_log (THIS->name, GF_LOG_ERROR,
@@ -1381,15 +1703,6 @@ glusterd_op_add_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        /* Need to reset the defrag/rebalance status accordingly */
-        switch (volinfo->defrag_status) {
-        case GF_DEFRAG_STATUS_FAILED:
-        case GF_DEFRAG_STATUS_COMPLETE:
-                volinfo->defrag_status = 0;
-        default:
-                break;
-        }
-
         ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret)
                 goto out;
@@ -1404,45 +1717,74 @@ out:
 int
 glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
 {
-        int                 ret            = -1;
-        char               *volname        = NULL;
-        glusterd_volinfo_t *volinfo        = NULL;
-        char               *brick          = NULL;
-        int32_t             count          = 0;
-        int32_t             i              = 1;
-        char                key[256]       = {0,};
-        int32_t             flag           = 0;
-        char                err_str[4096]  = {0,};
-        int                 need_rebalance = 0;
-        int                 force          = 0;
-        gf1_op_commands     cmd            = 0;
-        int32_t             replica_count  = 0;
-        glusterd_brickinfo_t *brickinfo    = NULL;
-        glusterd_brickinfo_t *tmp          = NULL;
-        glusterd_conf_t      *priv         = NULL;
-        char                  pidfile[PATH_MAX];
+        int                     ret            = -1;
+        char                    *volname       = NULL;
+        glusterd_volinfo_t      *volinfo       = NULL;
+        char                    *brick         = NULL;
+        int32_t                 count          = 0;
+        int32_t                 i              = 1;
+        char                    key[256]       = {0,};
+        int32_t                 flag           = 0;
+        char                    err_str[4096]  = {0,};
+        int                     need_rebalance = 0;
+        int                     force          = 0;
+        gf1_op_commands         cmd            = 0;
+        int32_t                 replica_count  = 0;
+        glusterd_brickinfo_t    *brickinfo     = NULL;
+        glusterd_brickinfo_t    *tmp           = NULL;
+        char                    *task_id_str   = NULL;
+        xlator_t                *this          = NULL;
+        dict_t                  *bricks_dict   = NULL;
+        char                    *brick_tmpstr  = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "command", &flag);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get brick count");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get command");
                 goto out;
         }
         cmd = flag;
 
+        /* Set task-id, if available, in ctx dict for operations other than
+         * start
+         */
+        if (is_origin_glusterd (dict) && (cmd != GF_OP_CMD_START)) {
+                if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                        ret = glusterd_copy_uuid_to_dict
+                                (volinfo->rebal.rebalance_id, dict,
+                                 GF_REMOVE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set remove-brick-id");
+                                goto out;
+                        }
+                }
+        }
+
+        /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
+         * remove-brick */
+        if ((cmd != GF_OP_CMD_START) || (cmd != GF_OP_CMD_STATUS)) {
+                uuid_clear (volinfo->rebal.rebalance_id);
+                volinfo->rebal.op = GD_OP_NONE;
+                dict_unref (volinfo->rebal.dict);
+                volinfo->rebal.dict = NULL;
+        }
+
         ret = -1;
         switch (cmd) {
         case GF_OP_CMD_NONE:
@@ -1452,47 +1794,49 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 ret = 0;
                 goto out;
 
-        case GF_OP_CMD_PAUSE:
-        {
-                if (volinfo->decommission_in_progress) {
-                        if (volinfo->defrag) {
-                                LOCK (&volinfo->defrag->lock);
-
-                                //volinfo->defrag_status = GF_DEFRAG_STATUS_PAUSED;
-
-                                UNLOCK (&volinfo->defrag->lock);
-                        }
-                }
-
-                /* no need to update anything */
-                ret = 0;
-                goto out;
-        }
-
-        case GF_OP_CMD_ABORT:
+        case GF_OP_CMD_STOP:
         {
-                if (volinfo->decommission_in_progress) {
-                        priv = THIS->private;
-                        if (!priv)
-                                return ret;
-
-                        GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
-
-                        glusterd_service_stop ("rebalance", pidfile, SIGTERM, 1);
-
-                }
-
                 /* Fall back to the old volume file */
-                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks, brick_list) {
+                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
+                                          brick_list) {
                         if (!brickinfo->decommissioned)
                                 continue;
                         brickinfo->decommissioned = 0;
                 }
+                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to create volfiles");
+                        goto out;
+                }
+
+                ret = glusterd_store_volinfo (volinfo,
+                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to store volinfo");
+                        goto out;
+                }
+
                 ret = 0;
-                break;
+                goto out;
         }
 
         case GF_OP_CMD_START:
+                /* Reset defrag status to 'NOT STARTED' whenever a
+                 * remove-brick/rebalance command is issued to remove
+                 * stale information from previous run.
+                 */
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
+                ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Missing remove-brick-id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
+                        volinfo->rebal.op = GD_OP_REMOVE_BRICK;
+                }
                 force = 0;
                 break;
 
@@ -1503,13 +1847,14 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
         case GF_OP_CMD_COMMIT_FORCE:
 
                 if (volinfo->decommission_in_progress) {
-                        if (volinfo->defrag) {
-                                LOCK (&volinfo->defrag->lock);
+                        if (volinfo->rebal.defrag) {
+                                LOCK (&volinfo->rebal.defrag->lock);
                                 /* Fake 'rebalance-complete' so the graph change
                                    happens right away */
-                                volinfo->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+                                volinfo->rebal.defrag_status =
+                                                GF_DEFRAG_STATUS_COMPLETE;
 
-                                UNLOCK (&volinfo->defrag->lock);
+                                UNLOCK (&volinfo->rebal.defrag->lock);
                         }
                         /* Graph change happens in rebalance _cbk function,
                            no need to do anything here */
@@ -1527,30 +1872,70 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-
+        /* Save the list of bricks for later usage only on starting a
+         * remove-brick. Right now this is required for displaying the task
+         * parameters with task status in volume status.
+         */
+        if (GF_OP_CMD_START == cmd) {
+                bricks_dict = dict_new ();
+                if (!bricks_dict) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_int32 (bricks_dict, "count", count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to save remove-brick count");
+                        goto out;
+                }
+        }
         while ( i <= count) {
                 snprintf (key, 256, "brick%d", i);
                 ret = dict_get_str (dict, key, &brick);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get %s", key);
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to get %s",
+                                key);
                         goto out;
                 }
 
+                if (GF_OP_CMD_START == cmd) {
+                        brick_tmpstr = gf_strdup (brick);
+                        if (!brick_tmpstr) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to duplicate brick name");
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to add brick to dict");
+                                goto out;
+                        }
+                        brick_tmpstr = NULL;
+                }
+
                 ret = glusterd_op_perform_remove_brick (volinfo, brick, force,
-                                                        (i == 1) ? &need_rebalance : NULL);
+                                                        &need_rebalance);
                 if (ret)
                         goto out;
                 i++;
         }
+        if (GF_OP_CMD_START == cmd)
+                volinfo->rebal.dict = dict_ref (bricks_dict);
+
         ret = dict_get_int32 (dict, "replica-count", &replica_count);
         if (!ret) {
-                gf_log (THIS->name, GF_LOG_INFO,
+                gf_log (this->name, GF_LOG_INFO,
                         "changing replica count %d to %d on volume %s",
                         volinfo->replica_count, replica_count,
                         volinfo->volname);
                 volinfo->replica_count = replica_count;
-                volinfo->dist_leaf_count = (volinfo->stripe_count *
-                                            replica_count);
+                volinfo->sub_count = replica_count;
+                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+                volinfo->subvol_count = (volinfo->brick_count /
+                                         volinfo->dist_leaf_count);
+
                 if (replica_count == 1) {
                         if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
                                 volinfo->type = GF_CLUSTER_TYPE_NONE;
@@ -1566,34 +1951,46 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
 
         ret = glusterd_create_volfiles_and_notify_services (volinfo);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "failed to create volfiles");
+                gf_log (this->name, GF_LOG_WARNING, "failed to create volfiles");
                 goto out;
         }
 
         ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "failed to store volinfo");
+                gf_log (this->name, GF_LOG_WARNING, "failed to store volinfo");
+                goto out;
+        }
+
+        if (GF_OP_CMD_START == cmd &&
+                        volinfo->status == GLUSTERD_STATUS_STARTED) {
+                ret = glusterd_nodesvcs_handle_reconfigure (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                               "Unable to reconfigure NFS-Server");
                 goto out;
+                }
         }
 
         /* Need to reset the defrag/rebalance status accordingly */
-        switch (volinfo->defrag_status) {
+        switch (volinfo->rebal.defrag_status) {
         case GF_DEFRAG_STATUS_FAILED:
         case GF_DEFRAG_STATUS_COMPLETE:
-                volinfo->defrag_status = 0;
+                volinfo->rebal.defrag_status = 0;
         default:
                 break;
         }
         if (!force && need_rebalance) {
                 /* perform the rebalance operations */
-                ret = glusterd_handle_defrag_start (volinfo, err_str, 4096,
-                                                    GF_DEFRAG_CMD_START_FORCE,
-                                                    glusterd_remove_brick_migrate_cbk);
+                ret = glusterd_handle_defrag_start
+                        (volinfo, err_str, sizeof (err_str),
+                         GF_DEFRAG_CMD_START_FORCE,
+                         glusterd_remove_brick_migrate_cbk, GD_OP_REMOVE_BRICK);
+
                 if (!ret)
                         volinfo->decommission_in_progress = 1;
 
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to start the rebalance");
                 }
         } else {
@@ -1605,5 +2002,9 @@ out:
         if (ret && err_str[0] && op_errstr)
                 *op_errstr = gf_strdup (err_str);
 
+        GF_FREE (brick_tmpstr);
+        if (bricks_dict)
+                dict_unref (bricks_dict);
+
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
index 5e12e1996..9433a128e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -31,19 +21,202 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "run.h"
+#include "syscall.h"
 
 #include <signal.h>
 
+static int
+dict_get_param (dict_t *dict, char *key, char **param);
+
+struct gsync_config_opt_vals_ gsync_confopt_vals[] = {
+        {.op_name        = "change_detector",
+         .no_of_pos_vals = 2,
+         .case_sensitive = _gf_true,
+         .values         = {"xsync", "changelog"},
+        },
+        {.op_name        = "special_sync_mode",
+         .no_of_pos_vals = 2,
+         .case_sensitive = _gf_true,
+         .values         = {"partial", "recover"}
+        },
+        {.op_name        = "log-level",
+         .no_of_pos_vals = 5,
+         .case_sensitive = _gf_false,
+         .values         = {"critical", "error", "warning", "info", "debug"}
+        },
+        {.op_name        = "use-tarssh",
+         .no_of_pos_vals = 6,
+         .case_sensitive = _gf_false,
+         .values         = {"true", "false", "0", "1", "yes", "no"}
+        },
+        {.op_name = NULL,
+        },
+};
+
 static char *gsync_reserved_opts[] = {
-        "gluster-command",
+        "gluster-command-dir",
         "pid-file",
+        "remote-gsyncd"
         "state-file",
         "session-owner",
+        "state-socket-unencoded",
+        "socketdir",
+        "ignore-deletes",
+        "local-id",
+        "local-path",
+        "slave-id",
+        NULL
+};
+
+static char *gsync_no_restart_opts[] = {
+        "checkpoint",
         NULL
 };
 
 int
-glusterd_handle_gsync_set (rpcsvc_request_t *req)
+__glusterd_handle_sys_exec (rpcsvc_request_t *req)
+{
+        int32_t                 ret     = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req              cli_req = {{0},};
+        glusterd_op_t           cli_op = GD_OP_SYS_EXEC;
+        glusterd_conf_t         *priv   = NULL;
+        char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+               } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        return ret;
+}
+
+int
+__glusterd_handle_copy_file (rpcsvc_request_t *req)
+{
+        int32_t                 ret     = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req              cli_req = {{0},};
+        glusterd_op_t           cli_op = GD_OP_COPY_FILE;
+        glusterd_conf_t         *priv   = NULL;
+        char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+               } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        return ret;
+}
+
+int
+__glusterd_handle_gsync_set (rpcsvc_request_t *req)
 {
         int32_t                 ret     = 0;
         dict_t                  *dict   = NULL;
@@ -55,15 +228,19 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
         int                     type = 0;
         glusterd_conf_t         *priv   = NULL;
         char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
 
         GF_ASSERT (req);
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
 
-        priv = THIS->private;
+       this = THIS;
+       GF_ASSERT (this);
+       priv = this->private;
+       GF_ASSERT (priv);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -77,17 +254,19 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
                 }
 
-                host_uuid = gf_strdup (uuid_utoa(priv->uuid));
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
                 if (host_uuid == NULL) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "failed to get"
-                                "the uuid of the host machine");
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
                         ret = -1;
                         goto out;
                 }
@@ -99,26 +278,31 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "master", &master);
         if (ret < 0) {
-                gf_log ("", GF_LOG_INFO, "master not found, while handling"
-                         GEOREP" options");
+                gf_log (this->name, GF_LOG_INFO, "master not found, while "
+                        "handling "GEOREP" options");
                 master = "(No Master)";
         }
 
         ret = dict_get_str (dict, "slave", &slave);
         if (ret < 0) {
-                gf_log ("", GF_LOG_INFO, "slave not not found, while"
+                gf_log (this->name, GF_LOG_INFO, "slave not found, while "
                         "handling "GEOREP" options");
                 slave = "(No Slave)";
         }
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "command type not found, while"
-                        "handling "GEOREP" options");
+                snprintf (err_str, sizeof (err_str), "Command type not found "
+                          "while handling "GEOREP" options");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         switch (type) {
+        case GF_GSYNC_OPTION_TYPE_CREATE:
+                strncpy (operation, "create", sizeof (operation));
+                cli_op = GD_OP_GSYNC_CREATE;
+                break;
 
         case GF_GSYNC_OPTION_TYPE_START:
                 strncpy (operation, "start", sizeof (operation));
@@ -135,30 +319,38 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
         case GF_GSYNC_OPTION_TYPE_STATUS:
                 strncpy (operation, "status", sizeof (operation));
                 break;
-        case GF_GSYNC_OPTION_TYPE_ROTATE:
-                strncpy (operation, "rotate", sizeof(operation));
-                break;
         }
 
-        gf_cmd_log ("volume "GEOREP, " %s command on %s,%s", operation, master,
-                    slave);
-        ret = glusterd_op_begin (req, GD_OP_GSYNC_SET, dict);
-        gf_cmd_log ("volume "GEOREP, " %s command on %s,%s %s  ", operation,
-                    master, slave, (ret != 0)? "FAILED" : "SUCCEEDED");
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
         return ret;
 }
 
+int
+glusterd_handle_sys_exec (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_sys_exec);
+}
+
+int
+glusterd_handle_copy_file (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_copy_file);
+}
+
+int
+glusterd_handle_gsync_set (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_gsync_set);
+}
 
 /*****
  *
@@ -180,7 +372,7 @@ glusterd_urltransform_add (runner_t *runner, const char *url)
         runner_add_arg (runner, url);
 }
 
-static void
+static int
 _glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *data)
 {
         runner_t *runner = (runner_t *)data;
@@ -190,6 +382,8 @@ _glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *d
         GF_ASSERT (slave);
         slave++;
         runner_add_arg (runner, slave);
+
+        return 0;
 }
 
 static void
@@ -228,6 +422,7 @@ glusterd_urltransform (runner_t *runner, char ***linearrp)
 
         arr_idx = 0;
         for (;;) {
+                size_t len;
                 line = GF_MALLOC (1024, gf_gld_mt_linebuf);
                 if (!line) {
                         error = _gf_true;
@@ -238,21 +433,24 @@ glusterd_urltransform (runner_t *runner, char ***linearrp)
                     NULL)
                         break;
 
-                if (line[strlen (line) - 1] != '\n') {
+                len = strlen (line);
+                if (len == 0 || line[len - 1] != '\n') {
                         GF_FREE (line);
                         error = _gf_true;
                         goto out;
                 }
-                line[strlen (line) - 1] = '\0';
+                line[len - 1] = '\0';
 
                 if (arr_idx == arr_len) {
+                        void *p = linearr;
                         arr_len <<= 1;
-                        linearr = GF_REALLOC (linearr, arr_len);
-                        if (!linearr) {
+                        p = GF_REALLOC (linearr, arr_len);
+                        if (!p) {
                                 GF_FREE (line);
                                 error = _gf_true;
                                 goto out;
                         }
+                        linearr = p;
                 }
                 linearr[arr_idx] = line;
 
@@ -300,7 +498,7 @@ struct dictidxmark {
         char *ikey;
 };
 
-static void
+static int
 _dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
 {
         struct dictidxmark *dim = data;
@@ -309,6 +507,7 @@ _dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
                 dim->ikey = key;
 
         dim->ithis++;
+        return 0;
 }
 
 static char *
@@ -354,9 +553,9 @@ glusterd_get_slave (glusterd_volinfo_t *vol, const char *slaveurl, char **slavek
 
 
 static int
-glusterd_query_extutil (char *resbuf, runner_t *runner)
+glusterd_query_extutil_generic (char *resbuf, size_t blen, runner_t *runner, void *data,
+                                int (*fcbk)(char *resbuf, size_t blen, FILE *fp, void *data))
 {
-        char               *ptr = NULL;
         int                 ret = 0;
 
         runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
@@ -366,92 +565,117 @@ glusterd_query_extutil (char *resbuf, runner_t *runner)
                 return -1;
         }
 
-        ptr = fgets(resbuf, PATH_MAX, runner_chio (runner, STDOUT_FILENO));
-        if (ptr)
-                resbuf[strlen(resbuf)-1] = '\0'; //strip off \n
+        ret = fcbk (resbuf, blen, runner_chio (runner, STDOUT_FILENO), data);
 
-        ret = runner_end (runner);
+        ret |= runner_end (runner);
         if (ret)
                 gf_log ("", GF_LOG_ERROR, "reading data from child failed");
 
         return ret ? -1 : 0;
 }
 
-int
-glusterd_gsync_get_param_file (char *prmfile, const char *param, char *master,
-                               char *slave, char *gl_workdir)
+static int
+_fcbk_singleline(char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        runner_t            runner = {0,};
+        char *ptr = NULL;
+
+        errno = 0;
+        ptr = fgets (resbuf, blen, fp);
+        if (ptr) {
+                size_t len = strlen(resbuf);
+                if (len && resbuf[len-1] == '\n')
+                        resbuf[len-1] = '\0'; //strip off \n
+        }
 
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, gl_workdir);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--config-get", NULL);
-        runner_argprintf (&runner, "%s-file", param);
+        return errno ? -1 : 0;
+}
 
-        return glusterd_query_extutil (prmfile, &runner);
+static int
+glusterd_query_extutil (char *resbuf, runner_t *runner)
+{
+        return glusterd_query_extutil_generic (resbuf, PATH_MAX, runner, NULL,
+                                               _fcbk_singleline);
 }
 
-int
-glusterd_gsync_get_session_owner (char *master, char *slave, char *session_owner,
-                                  char *gl_workdir)
+static int
+_fcbk_conftodict (char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        runner_t runner = {0,};
+        char   *ptr = NULL;
+        dict_t *dict = data;
+        char   *v = NULL;
 
-        runinit(&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, gl_workdir);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--config-get", "session-owner",
-                          NULL);
+        for (;;) {
+                errno = 0;
+                ptr = fgets (resbuf, blen, fp);
+                if (!ptr)
+                        break;
+                v = resbuf + strlen(resbuf) - 1;
+                while (isspace (*v))
+                        /* strip trailing space */
+                        *v-- = '\0';
+                if (v == resbuf)
+                        /* skip empty line */
+                        continue;
+                v = strchr (resbuf, ':');
+                if (!v)
+                        return -1;
+                *v++ = '\0';
+                while (isspace (*v))
+                        v++;
+                v = gf_strdup (v);
+                if (!v)
+                        return -1;
+                if (dict_set_dynstr (dict, resbuf, v) != 0) {
+                        GF_FREE (v);
+                        return -1;
+                }
+        }
 
-        return glusterd_query_extutil (session_owner, &runner);
+        return errno ? -1 : 0;
 }
 
-int
-glusterd_gsync_get_slave_log_file (char *master, char *slave, char *log_file)
+static int
+glusterd_gsync_get_config (char *master, char *slave, char *conf_path, dict_t *dict)
 {
-        int              ret        = -1;
-        runner_t         runner     = {0,};
-        char uuid_str[64]           = {0,};
-        glusterd_conf_t *priv       = NULL;
-        char            *gl_workdir = NULL;
-
-        GF_ASSERT(THIS);
-        GF_ASSERT(THIS->private);
+        /* key + value, where value must be able to accommodate a path */
+        char resbuf[256 + PATH_MAX] = {0,};
+        runner_t             runner = {0,};
 
-        priv = THIS->private;
-
-        GF_VALIDATE_OR_GOTO("gsyncd", master, out);
-        GF_VALIDATE_OR_GOTO("gsyncd", slave, out);
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, "--config-get-all", NULL);
 
-        gl_workdir = priv->workdir;
+        return glusterd_query_extutil_generic (resbuf, sizeof (resbuf),
+                                               &runner, dict, _fcbk_conftodict);
+}
 
-        /* get the session owner for the master-slave session */
-        ret = glusterd_gsync_get_session_owner (master, slave, uuid_str,
-                                                gl_workdir);
-        if (ret)
-                goto out;
+static int
+glusterd_gsync_get_param_file (char *prmfile, const char *param, char *master,
+                               char *slave, char *conf_path)
+{
+        runner_t            runner = {0,};
 
-        /* get the log file for the slave */
-        runinit(&runner);
+        runinit (&runner);
         runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, gl_workdir);
-        runner_argprintf (&runner, "--session-owner=%s", uuid_str);
-        runner_add_args  (&runner, slave, "--config-get", "log-file", NULL);
-
-        ret = glusterd_query_extutil (log_file, &runner);
+        runner_argprintf (&runner, "%s", conf_path);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, "--config-get", NULL);
+        runner_argprintf (&runner, "%s-file", param);
 
- out:
-        return ret;
+        return glusterd_query_extutil (prmfile, &runner);
 }
 
 static int
-gsyncd_getpidfile (char *master, char *slave, char *pidfile)
+gsyncd_getpidfile (char *master, char *slave, char *pidfile, char *conf_path)
 {
         int                ret             = -1;
         glusterd_conf_t    *priv  = NULL;
+        char               *confpath = NULL;
+        char                conf_buf[PATH_MAX] = "";
+        struct stat         stbuf = {0,};
+
 
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
@@ -461,8 +685,22 @@ gsyncd_getpidfile (char *master, char *slave, char *pidfile)
         GF_VALIDATE_OR_GOTO ("gsync", master, out);
         GF_VALIDATE_OR_GOTO ("gsync", slave, out);
 
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Using passed config template(%s).",
+                        conf_path);
+                confpath = conf_path;
+        } else {
+                ret = snprintf (conf_buf, sizeof(conf_buf) - 1,
+                                "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+                conf_buf[ret] = '\0';
+                confpath = conf_buf;
+                gf_log ("", GF_LOG_DEBUG, "Using default config template(%s).",
+                        confpath);
+        }
+
         ret = glusterd_gsync_get_param_file (pidfile, "pid", master,
-                                              slave, priv->workdir);
+                                              slave, confpath);
         if (ret == -1) {
                 ret = -2;
                 gf_log ("", GF_LOG_WARNING, "failed to create the pidfile string");
@@ -476,32 +714,6 @@ gsyncd_getpidfile (char *master, char *slave, char *pidfile)
 }
 
 static int
-glusterd_gsyncd_getlogfile (char *master, char *slave, char *log_file)
-{
-        int                ret             = -1;
-        glusterd_conf_t    *priv  = NULL;
-
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
-
-        priv = THIS->private;
-
-        GF_VALIDATE_OR_GOTO ("gsync", master, out);
-        GF_VALIDATE_OR_GOTO ("gsync", slave, out);
-
-        ret = glusterd_gsync_get_param_file (log_file, "log", master,
-                                              slave, priv->workdir);
-        if (ret == -1) {
-                ret = -2;
-                gf_log ("", GF_LOG_WARNING, "failed to gsyncd logfile");
-                goto out;
-        }
-
- out:
-        return ret;
-}
-
-static int
 gsync_status_byfd (int fd)
 {
         GF_ASSERT (fd >= -1);
@@ -518,23 +730,24 @@ gsync_status_byfd (int fd)
  * return -1 when not running
  */
 int
-gsync_status (char *master, char *slave, int *status)
+gsync_status (char *master, char *slave, char *conf_path, int *status)
 {
         char pidfile[PATH_MAX] = {0,};
         int  fd                = -1;
 
-        fd = gsyncd_getpidfile (master, slave, pidfile);
+        fd = gsyncd_getpidfile (master, slave, pidfile, conf_path);
         if (fd == -2)
                 return -1;
 
         *status = gsync_status_byfd (fd);
 
-        close (fd);
+        sys_close (fd);
+
         return 0;
 }
 
 
-int32_t
+static int32_t
 glusterd_gsync_volinfo_dict_set (glusterd_volinfo_t *volinfo,
                                  char *key, char *value)
 {
@@ -558,17 +771,49 @@ out:
         return 0;
 }
 
-int
-gsync_verify_config_options (dict_t *dict, char **op_errstr)
+static int
+glusterd_verify_gsyncd_spawn (char *master, char *slave)
+{
+        int                 ret = 0;
+        runner_t            runner = {0,};
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd",
+                         "--verify", "spawning", NULL);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "spawning child failed");
+                ret = -1;
+                goto out;
+        }
+
+        if (runner_end (&runner) != 0)
+                ret = -1;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+static int
+gsync_verify_config_options (dict_t *dict, char **op_errstr, char *volname)
 {
-        char  **resopt    = NULL;
-        int     i         = 0;
-        char   *subop     = NULL;
-        char   *slave     = NULL;
-        char   *op_name   = NULL;
-        char   *op_value  = NULL;
-        char   *t         = NULL;
-        gf_boolean_t banned = _gf_true;
+        char                         **resopt           = NULL;
+        int                            i                = 0;
+        int                            ret              = -1;
+        char                          *subop            = NULL;
+        char                          *slave            = NULL;
+        char                          *op_name          = NULL;
+        char                          *op_value         = NULL;
+        char                          *t                = NULL;
+        char                           errmsg[PATH_MAX] = "";
+        gf_boolean_t                   banned           = _gf_true;
+        gf_boolean_t                   op_match         = _gf_true;
+        gf_boolean_t                   val_match        = _gf_true;
+        struct gsync_config_opt_vals_ *conf_vals        = NULL;
 
         if (dict_get_str (dict, "subop", &subop) != 0) {
                 gf_log ("", GF_LOG_WARNING, "missing subop");
@@ -592,6 +837,12 @@ gsync_verify_config_options (dict_t *dict, char **op_errstr)
         }
 
         if (runcmd (GSYNCD_PREFIX"/gsyncd", "--config-check", op_name, NULL)) {
+                ret = glusterd_verify_gsyncd_spawn (volname, slave);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to spawn gsyncd");
+                        return 0;
+                }
+
                 gf_log ("", GF_LOG_WARNING, "Invalid option %s", op_name);
                 *op_errstr = gf_strdup ("Invalid option");
 
@@ -636,34 +887,117 @@ gsync_verify_config_options (dict_t *dict, char **op_errstr)
                 }
         }
 
+        /* Check options in gsync_confopt_vals for invalid values */
+        for (conf_vals = gsync_confopt_vals; conf_vals->op_name; conf_vals++) {
+                op_match = _gf_true;
+                for (i = 0; conf_vals->op_name[i] && op_name[i]; i++) {
+                        if (conf_vals->op_name[i] == op_name[i] ||
+                            (conf_vals->op_name[i] == '_' && op_name[i] == '-'))
+                                continue;
+                        op_match = _gf_false;
+                }
+
+                if (op_match) {
+                        if (!op_value)
+                                goto out;
+                        val_match = _gf_false;
+                        for (i = 0; i < conf_vals->no_of_pos_vals; i++) {
+                                if(conf_vals->case_sensitive){
+                                        if (!strcmp (conf_vals->values[i], op_value))
+                                                val_match = _gf_true;
+                                } else {
+                                        if (!strcasecmp (conf_vals->values[i], op_value))
+                                                val_match = _gf_true;
+                                }
+                        }
+
+                        if (!val_match) {
+                                ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                                "Invalid value(%s) for"
+                                                " option %s", op_value,
+                                                op_name);
+                                errmsg[ret] = '\0';
+
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                return -1;
+                        }
+                }
+        }
+out:
         return 0;
 }
 
-static void
+static int
+glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
+                                   char *slave, char *conf_path,
+                                   dict_t *rsp_dict, char *node);
+
+static int
 _get_status_mst_slv (dict_t *this, char *key, data_t *value, void *data)
 {
         glusterd_gsync_status_temp_t  *param = NULL;
         char                          *slave = NULL;
-        int                           ret = 0;
+        char                          *slave_buf = NULL;
+        char                          *slave_ip  = NULL;
+        char                          *slave_vol = NULL;
+        char                          *errmsg    = NULL;
+        char                           conf_path[PATH_MAX] = "";
+        int                           ret = -1;
+        glusterd_conf_t              *priv = NULL;
 
         param = (glusterd_gsync_status_temp_t *)data;
 
         GF_ASSERT (param);
         GF_ASSERT (param->volinfo);
 
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
         slave = strchr(value->data, ':');
-        if (slave)
-                slave ++;
-        else
-                return;
+        if (!slave)
+                return 0;
+        slave++;
+
+        ret = glusterd_get_slave_info (slave, &slave_ip, &slave_vol, &errmsg);
+        if (ret) {
+                if (errmsg)
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch "
+                                "slave details. Error: %s", errmsg);
+                else
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to fetch slave details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (conf_path, sizeof(conf_path) - 1,
+                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                        priv->workdir, param->volinfo->volname,
+                        slave_ip, slave_vol);
+        conf_path[ret] = '\0';
 
         ret = glusterd_get_gsync_status_mst_slv(param->volinfo,
-                                                slave, param->rsp_dict);
+                                                slave, conf_path,
+                                                param->rsp_dict,
+                                                param->node);
+out:
 
+        GF_FREE (errmsg);
+
+        if (slave_buf)
+                GF_FREE(slave_buf);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
+        return ret;
 }
 
 
-static void
+static int
 _get_max_gsync_slave_num (dict_t *this, char *key, data_t *value, void *data)
 {
         int  tmp_slvnum = 0;
@@ -672,25 +1006,30 @@ _get_max_gsync_slave_num (dict_t *this, char *key, data_t *value, void *data)
         sscanf (key, "slave%d", &tmp_slvnum);
         if (tmp_slvnum > *slvnum)
                 *slvnum = tmp_slvnum;
+
+        return 0;
 }
 
 static int
 glusterd_remove_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
                                char **op_errstr)
 {
+        int   zero_slave_entries = _gf_true;
         int   ret = 0;
         char *slavekey = NULL;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (slave);
 
-        ret = glusterd_get_slave (volinfo, slave, &slavekey);
-        if (ret < 0) {
-                ret++;
-                goto out;
-        }
-
-        dict_del (volinfo->gsync_slaves, slavekey);
+        do {
+                ret = glusterd_get_slave (volinfo, slave, &slavekey);
+                if (ret < 0 && zero_slave_entries) {
+                        ret++;
+                        goto out;
+                }
+                zero_slave_entries = _gf_false;
+                dict_del (volinfo->gsync_slaves, slavekey);
+        } while (ret >= 0);
 
         ret = glusterd_store_volinfo (volinfo,
                                       GLUSTERD_VOLINFO_VER_AC_INCREMENT);
@@ -743,8 +1082,9 @@ glusterd_gsync_get_uuid (char *slave, glusterd_volinfo_t *vol,
         return ret;
 }
 
-static int
+int
 glusterd_check_gsync_running_local (char *master, char *slave,
+                                    char *conf_path,
                                     gf_boolean_t *is_run)
 {
         int                 ret    = -1;
@@ -755,7 +1095,7 @@ glusterd_check_gsync_running_local (char *master, char *slave,
         GF_ASSERT (is_run);
 
         *is_run = _gf_false;
-        ret = gsync_status (master, slave, &ret_status);
+        ret = gsync_status (master, slave, conf_path, &ret_status);
         if (ret == 0 && ret_status == 0) {
                 *is_run = _gf_true;
         } else if (ret == -1) {
@@ -772,7 +1112,8 @@ glusterd_check_gsync_running_local (char *master, char *slave,
 
 static int
 glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
-                              char *host_uuid, char **op_errstr)
+                              char *host_uuid, char **op_errstr,
+                              gf_boolean_t is_force)
 {
         int    ret = 0;
         int    maxslv = 0;
@@ -795,7 +1136,8 @@ glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
         case -1:
                                 break;
         default:
-                GF_ASSERT (ret > 0);
+                if (!is_force)
+                        GF_ASSERT (ret > 0);
                 ret = dict_get_str (volinfo->gsync_slaves, slavekey, &slaveentry);
                 GF_ASSERT (ret == 0);
 
@@ -804,13 +1146,23 @@ glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
                  * assert an uuid mismatch
                  */
                 t = strtail (slaveentry, host_uuid);
-                GF_ASSERT (!t || *t != ':');
+                if (!is_force)
+                        GF_ASSERT (!t || *t != ':');
+
+                if (is_force) {
+                        gf_log ("", GF_LOG_DEBUG, GEOREP" has already been "
+                                "invoked for the %s (master) and %s (slave)."
+                                " Allowing without saving info again due to"
+                                " force command.", volinfo->volname, slave);
+                        ret = 0;
+                        goto out;
+                }
 
                 gf_log ("", GF_LOG_ERROR, GEOREP" has already been invoked for "
                                           "the %s (master) and %s (slave) "
                                           "from a different machine",
                                            volinfo->volname, slave);
-                *op_errstr = gf_strdup (GEOREP" already running in an an"
+                *op_errstr = gf_strdup (GEOREP" already running in "
                                         "another machine");
                 ret = -1;
                 goto out;
@@ -843,23 +1195,26 @@ glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
         return ret;
 }
 
-
 static int
 glusterd_op_verify_gsync_start_options (glusterd_volinfo_t *volinfo,
-                                        char *slave, char **op_errstr)
+                                        char *slave, char *conf_path,
+                                        char *statefile, char **op_errstr,
+                                        gf_boolean_t is_force)
 {
         int                     ret = -1;
         gf_boolean_t            is_running = _gf_false;
         char                    msg[2048] = {0};
         uuid_t                  uuid = {0};
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
+        glusterd_conf_t        *priv = NULL;
+        xlator_t               *this = NULL;
+        struct stat             stbuf = {0,};
 
         this = THIS;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (slave);
         GF_ASSERT (op_errstr);
+        GF_ASSERT (conf_path);
         GF_ASSERT (this && this->private);
 
         priv  = this->private;
@@ -869,26 +1224,56 @@ glusterd_op_verify_gsync_start_options (glusterd_volinfo_t *volinfo,
                           "before "GEOREP" start", volinfo->volname);
                 goto out;
         }
+
+        ret = lstat (statefile, &stbuf);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Session between %s and %s has"
+                          " not been created. Please create session and retry.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        /* Check if the gsync slave info is stored. If not
+         * session has not been created */
+        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Session between %s and %s has"
+                          " not been created. Please create session and retry.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        if (is_force) {
+                ret = 0;
+                goto out;
+        }
+
         /*Check if the gsync is already started in cmd. inited host
          * If so initiate add it into the glusterd's priv*/
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if ((ret == 0) && (uuid_compare (priv->uuid, uuid) == 0)) {
-                ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                          slave, &is_running);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), GEOREP" start option "
-                                  "validation failed ");
-                        goto out;
-                }
-                if (_gf_true == is_running) {
-                        snprintf (msg, sizeof (msg), GEOREP " session between"
-                                  " %s & %s already started", volinfo->volname,
-                                  slave);
-                        ret = -1;
-                        goto out;
-                }
+        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (ret) {
+                snprintf (msg, sizeof (msg), GEOREP" start option "
+                          "validation failed ");
+                goto out;
+        }
+        if (_gf_true == is_running) {
+                snprintf (msg, sizeof (msg), GEOREP " session between"
+                          " %s & %s already started", volinfo->volname,
+                          slave);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Unable to spawn gsyncd");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
         }
-        ret = 0;
 out:
         if (ret && (msg[0] != '\0')) {
                 *op_errstr = gf_strdup (msg);
@@ -912,13 +1297,168 @@ glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag)
         return 0;
 }
 
+/*
+ * is_geo_rep_active:
+ *      This function reads the state_file and sets is_active to 1 if the
+ *      monitor status is neither "Stopped" or "Not Started"
+ *
+ * RETURN VALUE:
+ *       0: On successful read of state_file.
+ *      -1: error.
+ */
+
+static int
+is_geo_rep_active (glusterd_volinfo_t *volinfo, char *slave,
+                   char *conf_path, int *is_active)
+{
+        dict_t                 *confd                      = NULL;
+        char                   *statefile                  = NULL;
+        char                   *master                     = NULL;
+        char                    monitor_status[PATH_MAX]   = "";
+        int                     ret                        = -1;
+        xlator_t               *this                       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        master = volinfo->volname;
+
+        confd = dict_new ();
+        if (!confd) {
+                gf_log ("", GF_LOG_ERROR, "Not able to create dict.");
+                goto out;
+        }
+
+        ret = glusterd_gsync_get_config (master, slave, conf_path,
+                                         confd);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data "
+                        "for %s(master), %s(slave)", master, slave);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_param (confd, "state_file", &statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_file's name "
+                        "for %s(master), %s(slave). Please check gsync "
+                        "config file.", master, slave);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
+                                              sizeof (monitor_status));
+        if (ret <= 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read the status "
+                        "file for %s(master), %s(slave)", master, slave);
+                strncpy (monitor_status, "defunct", sizeof (monitor_status));
+        }
+
+        if ((!strcmp(monitor_status, "Stopped")) ||
+            (!strcmp(monitor_status, "Not Started"))) {
+                *is_active = 0;
+        } else {
+                *is_active = 1;
+        }
+        ret = 0;
+out:
+        if (confd)
+                dict_destroy (confd);
+        return ret;
+}
+
+/*
+ * _get_slave_status:
+ *      Called for each slave in the volume from dict_foreach.
+ *      It calls is_geo_rep_active to get the monitor status.
+ *
+ * RETURN VALUE:
+ *      0: On successful read of state_file from is_geo_rep_active.
+ *         When it is found geo-rep is already active from previous calls.
+ *         When there is no slave.
+ *     -1: On error.
+ */
+
+int
+_get_slave_status (dict_t *dict, char *key, data_t *value, void *data)
+{
+        gsync_status_param_t          *param               = NULL;
+        char                          *slave               = NULL;
+        char                          *slave_ip            = NULL;
+        char                          *slave_vol           = NULL;
+        char                          *errmsg              = NULL;
+        char                           conf_path[PATH_MAX] = "";
+        int                            ret                 = -1;
+        glusterd_conf_t               *priv                = NULL;
+        xlator_t                      *this                = NULL;
+
+        param = (gsync_status_param_t *)data;
+
+        GF_ASSERT (param);
+        GF_ASSERT (param->volinfo);
+
+        if (param->is_active) {
+                ret = 0;
+                goto out;
+        }
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (this)
+                priv = this->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
+        slave = strchr(value->data, ':');
+        if (!slave) {
+                ret = 0;
+                goto out;
+        }
+        slave++;
+
+        ret = glusterd_get_slave_info (slave, &slave_ip, &slave_vol, &errmsg);
+        if (ret) {
+                if (errmsg)
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch "
+                                "slave details. Error: %s", errmsg);
+                else
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to fetch slave details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (conf_path, sizeof(conf_path) - 1,
+                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                        priv->workdir, param->volinfo->volname,
+                        slave_ip, slave_vol);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to assign conf_path.");
+                ret = -1;
+                goto out;
+        }
+        conf_path[ret] = '\0';
+
+        ret = is_geo_rep_active (param->volinfo,slave, conf_path,
+                                 &param->is_active);
+out:
+        GF_FREE(errmsg);
+        return ret;
+}
+
 static int
 glusterd_op_verify_gsync_running (glusterd_volinfo_t *volinfo,
-                                  char *slave, char **op_errstr)
+                                  char *slave, char *conf_path,
+                                  char **op_errstr)
 {
-        int                     ret = -1;
-        char                    msg[2048] = {0};
-        uuid_t                  uuid = {0};
+        int        pfd               = -1;
+        int        ret               = -1;
+        char       msg[2048]         = {0};
+        char       pidfile[PATH_MAX] = {0,};
 
         GF_ASSERT (THIS && THIS->private);
         GF_ASSERT (volinfo);
@@ -931,12 +1471,25 @@ glusterd_op_verify_gsync_running (glusterd_volinfo_t *volinfo,
 
                 goto out;
         }
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if (ret == -1) {
-                snprintf (msg, sizeof (msg), GEOREP" session between %s & %s"
-                          " not active", volinfo->volname, slave);
+
+        pfd = gsyncd_getpidfile (volinfo->volname, slave, pidfile, conf_path);
+        if (pfd == -2) {
+                gf_log ("", GF_LOG_ERROR, GEOREP" stop validation "
+                        "failed for %s & %s", volinfo->volname, slave);
+                ret = -1;
                 goto out;
         }
+        if (gsync_status_byfd (pfd) == -1) {
+                snprintf (msg, sizeof (msg), GEOREP" session b/w %s & %s is not"
+                          " running on this node.", volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+                /* monitor gsyncd already dead */
+                goto out;
+        }
+
+        if (pfd < 0)
+                goto out;
 
         ret = 0;
 out:
@@ -956,6 +1509,18 @@ glusterd_verify_gsync_status_opts (dict_t *dict, char **op_errstr)
         gf_boolean_t       exists = _gf_false;
         glusterd_volinfo_t *volinfo = NULL;
         int                ret = 0;
+        char               *conf_path = NULL;
+        char               *slave_ip = NULL;
+        char               *slave_vol = NULL;
+        glusterd_conf_t    *priv = NULL;
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
 
         ret = dict_get_str (dict, "master", &volname);
         if (ret < 0) {
@@ -980,38 +1545,191 @@ glusterd_verify_gsync_status_opts (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
- out:
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave  or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
-
 }
 
 
-static int
+int
 glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
-                            char **master, char **slave)
+                            char **master, char **slave, char **host_uuid)
 {
 
         int             ret = -1;
         GF_ASSERT (dict);
         GF_ASSERT (op_errstr);
-        GF_ASSERT (master);
-        GF_ASSERT (slave);
 
-        ret = dict_get_str (dict, "master", master);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "master not found");
-                *op_errstr = gf_strdup ("master not found");
+        if (master) {
+                ret = dict_get_str (dict, "master", master);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "master not found");
+                        *op_errstr = gf_strdup ("master not found");
+                        goto out;
+                }
+        }
+
+        if (slave) {
+                ret = dict_get_str (dict, "slave", slave);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "slave not found");
+                        *op_errstr = gf_strdup ("slave not found");
+                        goto out;
+                }
+        }
+
+        if (host_uuid) {
+                ret = dict_get_str (dict, "host-uuid", host_uuid);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "host_uuid not found");
+                        *op_errstr = gf_strdup ("host_uuid not found");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr)
+{
+        char             errmsg[PATH_MAX]          = "";
+        char            *command                   = NULL;
+        char             command_path[PATH_MAX]    = "";
+        struct stat      st                        = {0,};
+        int              ret                       = -1;
+        glusterd_conf_t *conf                      = NULL;
+        xlator_t        *this                      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "command", &command);
+        if (ret) {
+                strcpy (errmsg, "internal error");
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get command from dict");
                 goto out;
         }
 
-        ret = dict_get_str (dict, "slave", slave);
+        /* enforce local occurrence of the command */
+        if (strchr (command, '/')) {
+                strcpy (errmsg, "invalid command name");
+                ret = -1;
+                goto out;
+        }
+
+        sprintf (command_path, GSYNCD_PREFIX"/peer_%s", command);
+        /* check if it's executable */
+        ret = access (command_path, X_OK);
+        if (!ret)
+                /* check if it's a regular file */
+                ret = stat (command_path, &st);
+        if (!ret && !S_ISREG (st.st_mode))
+                ret = -1;
+
+out:
+        if (ret) {
+                if (errmsg[0] == '\0')
+                        snprintf (errmsg, sizeof (errmsg), "%s not found.",
+                                  command);
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr)
+{
+        char             abs_filename[PATH_MAX] = "";
+        char             errmsg[PATH_MAX]       = "";
+        char            *filename               = NULL;
+        char            *host_uuid              = NULL;
+        char             uuid_str [64]          = {0};
+        int              ret                    = -1;
+        glusterd_conf_t *priv                   = NULL;
+        struct stat      stbuf                  = {0,};
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
+
+        if (priv->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "host-uuid", &host_uuid);
         if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "slave not found");
-                *op_errstr = gf_strdup ("slave not found");
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                        " host-uuid from dict.");
                 goto out;
         }
 
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = dict_get_str (dict, "source", &filename);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                                " filename from dict.");
+                        *op_errstr = gf_strdup ("command unsuccessful");
+                        goto out;
+                }
+                snprintf (abs_filename, sizeof(abs_filename),
+                          "%s/%s", priv->workdir, filename);
+
+                ret = lstat (abs_filename, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                  " does not exist in %s", priv->workdir);
+                        *op_errstr = gf_strdup (errmsg);
+                        goto out;
+                }
+
+                if (!S_ISREG(stbuf.st_mode)) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                 " is not a regular file.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
 
         ret = 0;
 out:
@@ -1020,17 +1738,513 @@ out:
 }
 
 int
+glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
+                             char *conf_path, char **statefile)
+{
+        glusterd_conf_t *priv = NULL;
+        int              ret = -1;
+        char            *master    = NULL;
+        char            *buf       = NULL;
+        dict_t          *confd = NULL;
+        char            *confpath = NULL;
+        char             conf_buf[PATH_MAX] = "";
+        struct stat      stbuf = {0,};
+
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+        GF_ASSERT (volinfo);
+
+        master = volinfo->volname;
+
+        confd = dict_new ();
+        if (!confd) {
+                gf_log ("", GF_LOG_ERROR, "Unable to create new dict");
+                goto out;
+        }
+
+        priv = THIS->private;
+
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_INFO, "Using passed config template(%s).",
+                        conf_path);
+                confpath = conf_path;
+        } else {
+                ret = snprintf (conf_buf, sizeof(conf_buf) - 1,
+                                "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+                conf_buf[ret] = '\0';
+                confpath = conf_buf;
+                gf_log ("", GF_LOG_INFO, "Using default config template(%s).",
+                        confpath);
+        }
+
+        ret = glusterd_gsync_get_config (master, slave, confpath,
+                                         confd);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
+                        "for %s(master), %s(slave)", master, slave);
+                goto out;
+
+        }
+
+        ret = dict_get_param (confd, "state_file", &buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_file's name.");
+                goto out;
+        }
+
+        *statefile = gf_strdup(buf);
+        if (!*statefile) {
+                gf_log ("", GF_LOG_ERROR, "Unable to gf_strdup.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+ out:
+        if (confd)
+                dict_destroy (confd);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
+        return ret;
+}
+
+static int
+glusterd_create_status_file (char *master, char *slave, char *slave_ip,
+                             char *slave_vol, char *status)
+{
+        int                ret    = -1;
+        runner_t           runner = {0,};
+        glusterd_conf_t   *priv   = NULL;
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
+        if (!status) {
+                gf_log ("", GF_LOG_ERROR, "Status Empty");
+                goto out;
+        }
+        gf_log ("", GF_LOG_DEBUG, "slave = %s", slave);
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--create",
+                         status, "-c", NULL);
+        runner_argprintf (&runner, "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                          priv->workdir, master, slave_ip, slave_vol);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, NULL);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Creating status file failed.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_verify_slave (char *volname, char *slave_ip, char *slave,
+                       char **op_errstr, gf_boolean_t *is_force_blocker)
+{
+        int32_t          ret                     = -1;
+        runner_t         runner                  = {0,};
+        char             log_file_path[PATH_MAX] = "";
+        char             buf[PATH_MAX]           = "";
+        char            *tmp                     = NULL;
+        char            *save_ptr                = NULL;
+        glusterd_conf_t *priv                    = NULL;
+
+        GF_ASSERT (volname);
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (slave);
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
+        snprintf (log_file_path, sizeof(log_file_path),
+                  DEFAULT_LOG_FILE_DIRECTORY"/create_verify_log");
+
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gverify.sh", NULL);
+        runner_argprintf (&runner, "%s", volname);
+        runner_argprintf (&runner, "%s", slave_ip);
+        runner_argprintf (&runner, "%s", slave);
+        runner_argprintf (&runner, "%s", log_file_path);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Not a valid slave");
+                ret = glusterd_gsync_read_frm_status (log_file_path,
+                                                      buf, sizeof(buf));
+                if (ret <= 0) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to read from %s",
+                                log_file_path);
+                        goto out;
+                }
+
+                /* Tokenize the error message from gverify.sh to figure out
+                 * if the error is a force blocker or not. */
+                tmp = strtok_r (buf, "|", &save_ptr);
+                if (!strcmp (tmp, "FORCE_BLOCKER"))
+                        *is_force_blocker = 1;
+                else {
+                        /* No FORCE_BLOCKER flag present so all that is
+                         * present is the error message. */
+                        *is_force_blocker = 0;
+                        if (tmp)
+                                *op_errstr = gf_strdup (tmp);
+                        ret = -1;
+                        goto out;
+                }
+
+                /* Copy rest of the error message to op_errstr */
+                tmp = strtok_r (NULL, "|", &save_ptr);
+                if (tmp)
+                        *op_errstr = gf_strdup (tmp);
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        unlink (log_file_path);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mountbroker_check (char **slave_ip, char **op_errstr)
+{
+        int   ret             = -1;
+        char *tmp             = NULL;
+        char *save_ptr        = NULL;
+        char *username        = NULL;
+        char *host            = NULL;
+        char errmsg[PATH_MAX] = "";
+
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (*slave_ip);
+
+        /* Checking if hostname has user specified */
+        host = strstr (*slave_ip, "@");
+        if (!host) {
+                gf_log ("", GF_LOG_DEBUG, "No username provided.");
+                ret = 0;
+                goto out;
+        } else {
+                /* Moving the host past the '@' and checking if the
+                 * actual hostname also has '@' */
+                host++;
+                if (strstr (host, "@")) {
+                        gf_log ("", GF_LOG_DEBUG, "host = %s", host);
+                        ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                        "Invalid Hostname (%s).", host);
+                        errmsg[ret] = '\0';
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        if (op_errstr)
+                                *op_errstr = gf_strdup (errmsg);
+                        goto out;
+                }
+
+                /* Fetching the username and hostname
+                 * and checking if the username is non-root */
+                username = strtok_r (*slave_ip, "@", &save_ptr);
+                tmp = strtok_r (NULL, "@", &save_ptr);
+                if (strcmp (username, "root")) {
+                        ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                        "Non-root username (%s@%s) not allowed.",
+                                        username, tmp);
+                        errmsg[ret] = '\0';
+                        if (op_errstr)
+                                *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR,
+                                "Non-Root username not allowed.");
+                        ret = -1;
+                        goto out;
+                }
+
+                *slave_ip = gf_strdup (tmp);
+                if (!*slave_ip) {
+                        gf_log ("", GF_LOG_ERROR, "Out of memory");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr)
+{
+        char               *down_peerstr              = NULL;
+        char               *slave                     = NULL;
+        char               *volname                   = NULL;
+        char               *host_uuid                 = NULL;
+        char               *statefile                 = NULL;
+        char               *slave_ip                  = NULL;
+        char               *slave_vol                 = NULL;
+        char               *conf_path                 = NULL;
+        char                errmsg[PATH_MAX]          = "";
+        char                common_pem_file[PATH_MAX] = "";
+        char                hook_script[PATH_MAX]     = "";
+        char                uuid_str [64]             = "";
+        int                 ret                       = -1;
+        int                 is_pem_push               = -1;
+        gf_boolean_t        is_force                  = -1;
+        gf_boolean_t        is_force_blocker          = -1;
+        gf_boolean_t        exists                    = _gf_false;
+        glusterd_conf_t    *conf                      = NULL;
+        glusterd_volinfo_t *volinfo                   = NULL;
+        struct stat         stbuf                     = {0,};
+        xlator_t           *this                      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = glusterd_op_gsync_args_get (dict, op_errstr, &volname,
+                                          &slave, &host_uuid);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch arguments");
+                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                return -1;
+        }
+
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if ((ret) || (!exists)) {
+                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
+                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
+                          " exist", volname);
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                return -1;
+        }
+
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = glusterd_are_vol_all_peers_up (volinfo,
+                                                     &conf->peers,
+                                                     &down_peerstr);
+                if ((ret == _gf_false) && !is_force) {
+                        snprintf (errmsg, sizeof (errmsg), "Peer %s,"
+                                  " which is a part of %s volume, is"
+                                  " down. Please bring up the peer and"
+                                  " retry.", down_peerstr,
+                                  volinfo->volname);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        *op_errstr = gf_strdup (errmsg);
+                        GF_FREE (down_peerstr);
+                        down_peerstr = NULL;
+                        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                        return -1;
+                } else if (ret == _gf_false) {
+                        gf_log ("", GF_LOG_INFO, "Peer %s,"
+                                " which is a part of %s volume, is"
+                                " down. Force creating geo-rep session."
+                                " On bringing up the peer, re-run"
+                                " \"gluster system:: execute"
+                                " gsec_create\" and \"gluster volume"
+                                " geo-replication %s %s create push-pem"
+                                " force\"", down_peerstr, volinfo->volname,
+                                volinfo->volname, slave);
+                }
+
+                /* Checking if slave host is pingable, has proper passwordless
+                 * ssh login setup, slave volume is created, slave vol is empty,
+                 * and if it has enough memory and bypass in case of force if
+                 * the error is not a force blocker */
+                ret = glusterd_verify_slave (volname, slave_ip, slave_vol,
+                                             op_errstr, &is_force_blocker);
+                if (ret) {
+                        if (is_force && !is_force_blocker) {
+                                gf_log ("", GF_LOG_INFO, "%s is not a valid slave"
+                                        " volume. Error: %s. Force creating geo-rep"
+                                        " session.", slave, *op_errstr);
+                        } else {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "%s is not a valid slave volume. Error: %s",
+                                        slave, *op_errstr);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
+                if (!ret && is_pem_push) {
+                        ret = snprintf (common_pem_file,
+                                        sizeof(common_pem_file) - 1,
+                                        "%s"GLUSTERD_COMMON_PEM_PUB_FILE,
+                                        conf->workdir);
+                        common_pem_file[ret] = '\0';
+
+                        ret = snprintf (hook_script, sizeof(hook_script) - 1,
+                                        "%s"GLUSTERD_CREATE_HOOK_SCRIPT,
+                                        conf->workdir);
+                        hook_script[ret] = '\0';
+
+                        ret = lstat (common_pem_file, &stbuf);
+                        if (ret) {
+                                snprintf (errmsg, sizeof (errmsg), "%s"
+                                          " required for push-pem is"
+                                          " not present. Please run"
+                                          " \"gluster system:: execute"
+                                          " gsec_create\"", common_pem_file);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        ret = lstat (hook_script, &stbuf);
+                        if (ret) {
+                                snprintf (errmsg, sizeof (errmsg),
+                                          "The hook-script (%s) required "
+                                          "for push-pem is not present. "
+                                          "Please install the hook-script "
+                                          "and retry", hook_script);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        if (!S_ISREG(stbuf.st_mode)) {
+                                snprintf (errmsg, sizeof (errmsg), "%s"
+                                          " required for push-pem is"
+                                          " not a regular file. Please run"
+                                          " \"gluster system:: execute"
+                                          " gsec_create\"", common_pem_file);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        ret = glusterd_get_statefile_name (volinfo, slave, conf_path, &statefile);
+        if (ret) {
+                if (!strstr(slave, "::"))
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "%s is not a valid slave url.", slave);
+                else
+                        snprintf (errmsg, sizeof (errmsg), "Please check gsync "
+                                  "config file. Unable to get statefile's name");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "statefile", statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store statefile path");
+                goto out;
+        }
+
+        ret = lstat (statefile, &stbuf);
+        if (!ret && !is_force) {
+                snprintf (errmsg, sizeof (errmsg), "Session between %s"
+                          " and %s is already created.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        } else if (!ret)
+                gf_log ("", GF_LOG_INFO, "Session between %s"
+                        " and %s is already created. Force"
+                        " creating again.", volinfo->volname, slave);
+
+        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to spawn gsyncd.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        if (ret && errmsg[0] != '\0')
+                *op_errstr = gf_strdup (errmsg);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
 glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
 {
         int                     ret     = 0;
         int                     type    = 0;
         char                    *volname = NULL;
         char                    *slave   = NULL;
+        char                    *slave_ip = NULL;
+        char                    *slave_vol = NULL;
+        char                    *down_peerstr = NULL;
+        char                    *statefile    = NULL;
+        char                    *path_list    = NULL;
+        char                    *conf_path    = NULL;
         gf_boolean_t            exists   = _gf_false;
         glusterd_volinfo_t      *volinfo = NULL;
         char                    errmsg[PATH_MAX] = {0,};
         dict_t                  *ctx = NULL;
+        gf_boolean_t            is_force = 0;
+        gf_boolean_t            is_force_blocker = -1;
+        gf_boolean_t            is_running = _gf_false;
+        uuid_t                  uuid = {0};
+        char                    uuid_str [64] = {0};
+        char                    *host_uuid = NULL;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *conf = NULL;
+        struct stat             stbuf = {0,};
 
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret < 0) {
@@ -1039,25 +2253,26 @@ glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        switch (type) {
-        case GF_GSYNC_OPTION_TYPE_STATUS:
+        if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
                 ret = glusterd_verify_gsync_status_opts (dict, op_errstr);
-
                 goto out;
-        case GF_GSYNC_OPTION_TYPE_CONFIG:
-                ret = gsync_verify_config_options (dict, op_errstr);
+        }
 
+        ret = glusterd_op_gsync_args_get (dict, op_errstr,
+                                          &volname, &slave, &host_uuid);
+        if (ret)
                 goto out;
 
-        case GF_GSYNC_OPTION_TYPE_ROTATE:
-                /* checks same as status mode */
-                ret = glusterd_verify_gsync_status_opts(dict, op_errstr);
-                goto out;
-        }
+        uuid_utoa_r (MY_UUID, uuid_str);
 
-        ret = glusterd_op_gsync_args_get (dict, op_errstr, &volname, &slave);
-        if (ret)
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+               *op_errstr = gf_strdup (errmsg);
+                ret = -1;
                 goto out;
+        }
 
         exists = glusterd_check_volume_exists (volname);
         ret = glusterd_volinfo_find (volname, &volinfo);
@@ -1070,19 +2285,115 @@ glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_statefile_name (volinfo, slave, conf_path, &statefile);
+        if (ret) {
+                /* Checking if slave host is pingable, has proper passwordless
+                 * ssh login setup */
+                ret = glusterd_verify_slave (volname, slave_ip, slave_vol,
+                                             op_errstr, &is_force_blocker);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "%s is not a valid slave volume. Error: %s",
+                                slave, *op_errstr);
+                        goto out;
+                }
+
+                if (!strstr(slave, "::"))
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "%s is not a valid slave url.", slave);
+                else
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to get statefile's name");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "statefile", statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store statefile path");
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        /* Allowing stop force to bypass the statefile check
+         * as this command acts as a fail safe method to stop geo-rep
+         * session. */
+        if ((type == GF_GSYNC_OPTION_TYPE_CONFIG) ||
+            ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force) ||
+            (type == GF_GSYNC_OPTION_TYPE_DELETE)) {
+                ret = lstat (statefile, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
+                                  " session between %s and %s does not exist.",
+                                  volinfo->volname, slave);
+                        gf_log ("", GF_LOG_ERROR, "%s. statefile = %s",
+                                errmsg, statefile);
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check if all peers that are a part of the volume are up or not */
+        if ((type == GF_GSYNC_OPTION_TYPE_DELETE) ||
+            ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force)) {
+                if (!strcmp (uuid_str, host_uuid)) {
+                        ret = glusterd_are_vol_all_peers_up (volinfo,
+                                                             &conf->peers,
+                                                             &down_peerstr);
+                        if (ret == _gf_false) {
+                                snprintf (errmsg, sizeof (errmsg), "Peer %s,"
+                                          " which is a part of %s volume, is"
+                                          " down. Please bring up the peer and"
+                                          " retry.", down_peerstr,
+                                          volinfo->volname);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                GF_FREE (down_peerstr);
+                                down_peerstr = NULL;
+                                goto out;
+                        }
+                }
+        }
+
         switch (type) {
         case GF_GSYNC_OPTION_TYPE_START:
+                /* don't attempt to start gsync if replace-brick is
+                 * in progress */
+                if (glusterd_is_rb_ongoing (volinfo)) {
+                        snprintf (errmsg, sizeof(errmsg), "replace-brick is in"
+                                   " progress, not starting geo-replication");
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                }
+
                 ret = glusterd_op_verify_gsync_start_options (volinfo, slave,
-                                                              op_errstr);
+                                                              conf_path, statefile,
+                                                              op_errstr, is_force);
                 if (ret)
                         goto out;
                 ctx = glusterd_op_get_ctx();
                 if (ctx) {
-                        /*gsyncd does a fuse mount to start the geo-rep session*/
+                        /* gsyncd does a fuse mount to start
+                         * the geo-rep session */
                         if (!glusterd_is_fuse_available ()) {
-                                gf_log ("glusterd", GF_LOG_ERROR, "Unable to open"
-                                        " /dev/fuse (%s), geo-replication start"
-                                        " failed", strerror (errno));
+                                gf_log ("glusterd", GF_LOG_ERROR, "Unable to "
+                                        "open /dev/fuse (%s), geo-replication "
+                                        "start failed", strerror (errno));
                                 snprintf (errmsg, sizeof(errmsg),
                                           "fuse unvailable");
                                 *op_errstr = gf_strdup (errmsg);
@@ -1093,17 +2404,72 @@ glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
                 break;
 
         case GF_GSYNC_OPTION_TYPE_STOP:
-                ret = glusterd_op_verify_gsync_running (volinfo, slave,
-                                                        op_errstr);
+                if (!is_force) {
+                        ret = glusterd_op_verify_gsync_running (volinfo, slave,
+                                                                conf_path,
+                                                                op_errstr);
+                        if (ret) {
+                                ret = glusterd_get_local_brickpaths (volinfo,
+                                                                     &path_list);
+                                if (path_list)
+                                        ret = -1;
+                        }
+                }
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+                ret = gsync_verify_config_options (dict, op_errstr, volname);
+                goto out;
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_DELETE:
+                /* Check if the gsync session is still running
+                 * If so ask the user to stop geo-replication first.*/
+                ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+                if (ret) {
+                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
+                                  " session between %s and %s does not exist.",
+                                  volinfo->volname, slave);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                } else {
+                        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                                  slave, conf_path,
+                                                                  &is_running);
+                        if (_gf_true == is_running) {
+                                snprintf (errmsg, sizeof (errmsg), GEOREP
+                                          " session between %s & %s is "
+                                          "still active. Please stop the "
+                                          "session and retry.",
+                                          volinfo->volname, slave);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to spawn gsyncd");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                }
+
                 break;
         }
 
 out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int
-stop_gsync (char *master, char *slave, char **msg)
+static int
+stop_gsync (char *master, char *slave, char **msg,
+            char *conf_path, gf_boolean_t is_force)
 {
         int32_t         ret     = 0;
         int             pfd     = -1;
@@ -1115,23 +2481,23 @@ stop_gsync (char *master, char *slave, char **msg)
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
 
-        pfd = gsyncd_getpidfile (master, slave, pidfile);
-        if (pfd == -2) {
+        pfd = gsyncd_getpidfile (master, slave, pidfile, conf_path);
+        if (pfd == -2 && !is_force) {
                 gf_log ("", GF_LOG_ERROR, GEOREP" stop validation "
                         " failed for %s & %s", master, slave);
                 ret = -1;
                 goto out;
         }
-        if (gsync_status_byfd (pfd) == -1) {
+        if (gsync_status_byfd (pfd) == -1 && !is_force) {
                 gf_log ("", GF_LOG_ERROR, "gsyncd b/w %s & %s is not"
                         " running", master, slave);
-                if (msg)
-                        *msg = gf_strdup ("Warning: "GEOREP" session was in "
-                                          "corrupt  state");
                 /* monitor gsyncd already dead */
                 goto out;
         }
 
+        if (pfd < 0)
+                goto out;
+
         ret = read (pfd, buf, 1024);
         if (ret > 0) {
                 pid = strtol (buf, NULL, 10);
@@ -1158,17 +2524,101 @@ stop_gsync (char *master, char *slave, char **msg)
         ret = 0;
 
 out:
-        close (pfd);
+        sys_close (pfd);
+
+        if (is_force)
+                ret = 0;
         return ret;
 }
 
-int
-glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
-                                      dict_t *resp_dict);
+/*
+ * glusterd_gsync_op_already_set:
+ *      This funcion checks whether the op_value is same as in the
+ *      gsyncd.conf file.
+ *
+ * RETURN VALUE:
+ *      0 : op_value matches the conf file.
+ *      1 : op_value does not matches the conf file or op_param not
+ *          found in conf file.
+ *     -1 : error
+ */
 
 int
+glusterd_gsync_op_already_set (char* master, char* slave, char* conf_path,
+                                             char* op_name, char* op_value)
+{
+        dict_t                 *confd                       = NULL;
+        char                   *op_val_buf                  = NULL;
+        int32_t                 op_val_conf                 = 0;
+        int32_t                 op_val_cli                  = 0;
+        int32_t                 ret                         = -1;
+        gf_boolean_t            is_bool                     = _gf_true;
+
+        confd = dict_new ();
+        if (!confd) {
+                gf_log ("", GF_LOG_ERROR, "Not able to create dict.");
+                return -1;
+        }
+
+        ret = glusterd_gsync_get_config (master, slave, conf_path,
+                                         confd);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
+                            "for %s(master), %s(slave)", master, slave);
+                goto out;
+        }
+
+        ret = dict_get_param (confd, op_name, &op_val_buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get op_value "
+                            "for %s(master), %s(slave). Please check gsync "
+                            "config file.", master, slave);
+                ret = 1;
+                goto out;
+        }
+
+        gf_log("",GF_LOG_DEBUG, "val_cli:%s  val_conf:%s",op_value,op_val_buf);
+
+        if (!strcmp(op_val_buf,"true") || !strcmp(op_val_buf,"1")
+            || !strcmp(op_val_buf,"yes")) {
+                op_val_conf = 1;
+        } else if(!strcmp(op_val_buf,"false") || !strcmp(op_val_buf,"0")
+                || !strcmp(op_val_buf,"no")) {
+                op_val_conf = 0;
+        } else {
+                is_bool = _gf_false;
+        }
+
+        if (is_bool) {
+                if (!strcmp(op_value,"true") || !strcmp(op_value,"1")
+                    || !strcmp(op_value,"yes")) {
+                        op_val_cli = 1;
+                } else {
+                        op_val_cli = 0;
+                }
+
+                if ( op_val_cli == op_val_conf ) {
+                        ret = 0;
+                        goto out;
+                }
+        } else {
+                if (!strcmp(op_val_buf,op_value)) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        ret = 1;
+
+out:
+        dict_unref(confd);
+        return ret;
+}
+
+static int
 glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
-                          dict_t *dict, dict_t *resp_dict, char **op_errstr)
+                          char *path_list, dict_t *dict,
+                          dict_t *resp_dict, char **op_errstr)
 {
         int32_t         ret     = -1;
         char            *op_name = NULL;
@@ -1177,6 +2627,13 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
         glusterd_conf_t *priv   = NULL;
         char            *subop  = NULL;
         char            *master = NULL;
+        char            *conf_path = NULL;
+        char            *slave_ip  = NULL;
+        char            *slave_vol = NULL;
+        struct stat      stbuf     = {0, };
+        gf_boolean_t     restart_required = _gf_true;
+        char           **resopt    = NULL;
+        gf_boolean_t     op_already_set = _gf_false;
 
         GF_ASSERT (slave);
         GF_ASSERT (op_errstr);
@@ -1211,10 +2668,17 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
                 goto out;
         }
 
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
         master = "";
         runinit (&runner);
         runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, priv->workdir);
+        runner_argprintf (&runner, "%s", conf_path);
         if (volinfo) {
                 master = volinfo->volname;
                 runner_argprintf (&runner, ":%s", master);
@@ -1224,7 +2688,27 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
         runner_add_arg (&runner, op_name);
         if (op_value)
                 runner_add_arg (&runner, op_value);
+
+        if ( strcmp(op_name,"checkpoint") != 0 ) {
+                ret = glusterd_gsync_op_already_set(master,slave,conf_path,
+                                                           op_name,op_value);
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_WARNING,
+                                "glusterd_gsync_op_already_set failed.");
+                        gf_asprintf (op_errstr, GEOREP" config-%s failed for "
+                                     "%s %s", subop, master, slave);
+                        goto out;
+                }
+                if (ret == 0) {
+                        gf_log("", GF_LOG_DEBUG, "op_value is already set");
+                        op_already_set = _gf_true;
+                        goto out;
+                }
+        }
+
+        synclock_unlock (&priv->big_lock);
         ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret) {
                 gf_log ("", GF_LOG_WARNING, "gsyncd failed to "
                         "%s %s option for %s %s peers",
@@ -1235,15 +2719,58 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
 
                 goto out;
         }
+
+        if ((!strcmp (op_name, "state_file")) && (op_value)) {
+
+                ret = lstat (op_value, &stbuf);
+                if (ret) {
+                        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to fetch slave IP.");
+                                goto out;
+                        }
+
+                        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to fetch slave volume name.");
+                                goto out;
+                        }
+
+                        ret = glusterd_create_status_file (volinfo->volname, slave,
+                                                           slave_ip, slave_vol,
+                                                           "Switching Status File");
+                        if (ret || lstat (op_value, &stbuf)) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to create %s"
+                                        ". Error : %s", op_value,
+                                        strerror (errno));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
         ret = 0;
         gf_asprintf (op_errstr, "config-%s successful", subop);
 
 out:
-        if (!ret && volinfo) {
+        if (!ret && volinfo && !op_already_set) {
+            for (resopt = gsync_no_restart_opts; *resopt; resopt++) {
+                restart_required = _gf_true;
+                if (!strcmp ((*resopt), op_name)){
+                    restart_required = _gf_false;
+                    break;
+                }
+            }
+
+            if (restart_required) {
                 ret = glusterd_check_restart_gsync_session (volinfo, slave,
-                                                            resp_dict);
+                                                            resp_dict, path_list,
+                                                            conf_path, 0);
                 if (ret)
-                        *op_errstr = gf_strdup ("internal error");
+                    *op_errstr = gf_strdup ("internal error");
+            }
         }
 
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
@@ -1251,113 +2778,445 @@ out:
 }
 
 int
-glusterd_gsync_read_frm_status (char *path, char *data)
+glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen)
 {
         int                 ret = 0;
-        FILE               *status_file = NULL;
+        int           status_fd = -1;
 
         GF_ASSERT (path);
-        GF_ASSERT (data);
-        status_file = fopen (path, "r");
-        if (status_file  == NULL) {
-                gf_log ("", GF_LOG_WARNING, "Unable to read gsyncd status"
+        GF_ASSERT (buf);
+        status_fd = open (path, O_RDONLY);
+        if (status_fd == -1) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read gsyncd status"
                         " file");
                 return -1;
         }
-        ret = fread (data, PATH_MAX, 1, status_file);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "Status file of gsyncd is corrupt");
+        ret = read (status_fd, buf, blen - 1);
+        if (ret > 0) {
+                size_t len = strnlen (buf, ret);
+                /* Ensure there is a NUL byte and that it's not the first.  */
+                if (len == 0 || len == blen - 1) {
+                        ret = -1;
+                } else {
+                        char *p = buf + len - 1;
+                        while (isspace (*p))
+                                *p-- = '\0';
+                }
+        } else if (ret < 0)
+                gf_log ("", GF_LOG_ERROR, "Status file of gsyncd is corrupt");
+
+        close (status_fd);
+        return ret;
+}
+
+static int
+dict_get_param (dict_t *dict, char *key, char **param)
+{
+        char  *dk = NULL;
+        char   *s = NULL;
+        char    x = '\0';
+        int   ret = 0;
+
+        if (dict_get_str (dict, key, param) == 0)
+                return 0;
+
+        dk = gf_strdup (key);
+        if (!key)
+                return -1;
+
+        s = strpbrk (dk, "-_");
+        if (!s)
                 return -1;
+        x = (*s == '-') ? '_' : '-';
+        *s++ = x;
+        while ((s = strpbrk (s, "-_")))
+                *s++ = x;
+
+        ret = dict_get_str (dict, dk, param);
+
+        GF_FREE (dk);
+        return ret;
+}
+
+static int
+glusterd_parse_gsync_status (char *buf, gf_gsync_status_t *sts_val)
+{
+        int              ret      = -1;
+        int              i      = -1;
+        int              num_of_fields = 8;
+        char            *token    = NULL;
+        char           **tokens   = NULL;
+        char           **ptr   = NULL;
+        char            *save_ptr = NULL;
+        char             na_buf[] = "N/A";
+
+        if (!buf) {
+                gf_log ("", GF_LOG_ERROR, "Empty buf");
+                goto out;
         }
 
-        data[strlen(data)-1] = '\0';
+        tokens = calloc (num_of_fields, sizeof (char *));
+        if (!tokens) {
+                gf_log ("", GF_LOG_ERROR, "Out of memory");
+                goto out;
+        }
 
-        return 0;
+        ptr = tokens;
+
+        for (token = strtok_r (buf, ",", &save_ptr); token;
+             token = strtok_r (NULL, ",", &save_ptr)) {
+                *ptr = gf_strdup(token);
+                if (!*ptr) {
+                        gf_log ("", GF_LOG_ERROR, "Out of memory");
+                        goto out;
+                }
+                ptr++;
+        }
+
+        for (i = 0; i < num_of_fields; i++) {
+                token = strtok_r (tokens[i], ":", &save_ptr);
+                token = strtok_r (NULL, "\0", &save_ptr);
+                token++;
+
+                /* token NULL check */
+                if (!token && (i != 0) &&
+                    (i != 5) && (i != 7))
+                    token = na_buf;
+
+                if (i == 0) {
+                        if (!token)
+                            token = na_buf;
+                        else {
+                            token++;
+                            if (!token)
+                                token = na_buf;
+                            else
+                                token[strlen(token) - 1] = '\0';
+                        }
+                        memcpy (sts_val->slave_node, token, strlen(token));
+                }
+                if (i == 1)
+                        memcpy (sts_val->files_syncd, token, strlen(token));
+                if (i == 2)
+                        memcpy (sts_val->purges_remaining, token, strlen(token));
+                if (i == 3)
+                        memcpy (sts_val->total_files_skipped, token, strlen(token));
+                if (i == 4)
+                        memcpy (sts_val->files_remaining, token, strlen(token));
+                if (i == 5) {
+                        if (!token)
+                            token = na_buf;
+                        else {
+                            token++;
+                            if (!token)
+                                token = na_buf;
+                            else
+                                token[strlen(token) - 1] = '\0';
+                        }
+                        memcpy (sts_val->worker_status, token, strlen(token));
+                }
+                if (i == 6)
+                        memcpy (sts_val->bytes_remaining, token, strlen(token));
+                if (i == 7) {
+                        if (!token)
+                            token = na_buf;
+                        else {
+                            token++;
+                            if (!token)
+                                token = na_buf;
+                            else
+                                token[strlen(token) - 2] = '\0';
+                        }
+                        memcpy (sts_val->crawl_status, token, strlen(token));
+                }
+        }
+
+        ret = 0;
+out:
+        for (i = 0; i< num_of_fields; i++)
+               if (tokens[i])
+                       GF_FREE(tokens[i]);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_gsync_fetch_status_extra (char *path, gf_gsync_status_t *sts_val)
+{
+        char sockpath[PATH_MAX] = {0,};
+        struct sockaddr_un   sa = {0,};
+        int                   s = -1;
+        struct pollfd       pfd = {0,};
+        int                 ret = 0;
+
+        glusterd_set_socket_filepath (path, sockpath, sizeof (sockpath));
+
+        strncpy(sa.sun_path, sockpath, sizeof(sa.sun_path));
+        if (sa.sun_path[sizeof (sa.sun_path) - 1])
+                return -1;
+        sa.sun_family = AF_UNIX;
+
+        s = socket(AF_UNIX, SOCK_STREAM, 0);
+        if (s == -1)
+                return -1;
+        ret = fcntl (s, F_GETFL);
+        if (ret != -1)
+                ret = fcntl (s, F_SETFL, ret | O_NONBLOCK);
+        if (ret == -1)
+                goto out;
+
+        ret = connect (s, (struct sockaddr *)&sa, sizeof (sa));
+        if (ret == -1)
+                goto out;
+        pfd.fd = s;
+        pfd.events = POLLIN;
+        /* we don't want to hang on gsyncd */
+        if (poll (&pfd, 1, 5000) < 1 ||
+            !(pfd.revents & POLLIN)) {
+                ret = -1;
+                goto out;
+        }
+        ret = read(s, sts_val->checkpoint_status,
+                   sizeof(sts_val->checkpoint_status));
+        /* we expect a terminating 0 byte */
+        if (ret == 0 || (ret > 0 && sts_val->checkpoint_status[ret - 1]))
+                ret = -1;
+        if (ret > 0) {
+                ret = 0;
+        }
+
+out:
+        close (s);
+        return ret;
 }
 
 int
-glusterd_read_status_file (char *master, char *slave,
-                           dict_t *dict)
-{
-        glusterd_conf_t  *priv = NULL;
-        int              ret = 0;
-        char             statusfile[PATH_MAX] = {0, };
-        char             buff[PATH_MAX] = {0, };
-        char             mst[PATH_MAX] = {0, };
-        char             slv[PATH_MAX] = {0, };
-        char             sts[PATH_MAX] = {0, };
-        int              gsync_count = 0;
-        int              status = 0;
+glusterd_read_status_file (glusterd_volinfo_t *volinfo, char *slave,
+                           char *conf_path, dict_t *dict, char *node)
+{
+        char                    brick_state_file[PATH_MAX] = "";
+        char                    brick_path[PATH_MAX]       = "";
+        char                   *georep_session_wrkng_dir   = NULL;
+        char                   *master                     = NULL;
+        char                    tmp[1024]                  = "";
+        char                    sts_val_name[1024]         = "";
+        char                    monitor_status[NAME_MAX]   = "";
+        char                   *statefile                  = NULL;
+        char                   *socketfile                 = NULL;
+        dict_t                 *confd                      = NULL;
+        int                     gsync_count                = 0;
+        int                     i                          = 0;
+        int                     ret                        = 0;
+        glusterd_brickinfo_t   *brickinfo                  = NULL;
+        gf_gsync_status_t      *sts_val                    = NULL;
+        glusterd_conf_t        *priv                       = NULL;
 
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
+        GF_ASSERT (volinfo);
+
+        master = volinfo->volname;
+
+        confd = dict_new ();
+        if (!dict) {
+                gf_log ("", GF_LOG_ERROR, "Not able to create dict.");
+                return -1;
+        }
 
         priv = THIS->private;
-        ret = glusterd_gsync_get_param_file (statusfile, "state", master,
-                                             slave, priv->workdir);
+
+        ret = glusterd_gsync_get_config (master, slave, conf_path,
+                                         confd);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to get the name of status"
-                        "file for %s(master), %s(slave)", master, slave);
+                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
+                        "for %s(master), %s(slave)", master, slave);
                 goto out;
 
         }
 
-        ret = gsync_status (master, slave, &status);
-        if (ret == 0 && status == -1) {
-                strncpy (buff, "corrupt", sizeof (buff));
-                goto done;
-        } else if (ret == -1)
+        ret = dict_get_param (confd, "state_file", &statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_file's name "
+                        "for %s(master), %s(slave). Please check gsync "
+                        "config file.", master, slave);
                 goto out;
+        }
 
-        ret = glusterd_gsync_read_frm_status (statusfile, buff);
-        if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to read the status"
+        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
+                                              sizeof (monitor_status));
+        if (ret <= 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read the status"
                         "file for %s(master), %s(slave)", master, slave);
+                strncpy (monitor_status, "defunct", sizeof (monitor_status));
+        }
+
+        ret = dict_get_param (confd, "georep_session_working_dir",
+                              &georep_session_wrkng_dir);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get geo-rep session's "
+                        "working directory name for %s(master), %s(slave). "
+                        "Please check gsync config file.", master, slave);
                 goto out;
+        }
 
+        ret = dict_get_param (confd, "state_socket_unencoded", &socketfile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get socket file's name "
+                        "for %s(master), %s(slave). Please check gsync "
+                        "config file.", master, slave);
+                goto out;
         }
 
- done:
         ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
-
         if (ret)
-                gsync_count = 1;
-        else
-                gsync_count++;
+                gsync_count = 0;
 
-        snprintf (mst, sizeof (mst), "master%d", gsync_count);
-        ret = dict_set_dynstr (dict, mst, gf_strdup (master));
-        if (ret)
-                goto out;
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
 
-        snprintf (slv, sizeof (slv), "slave%d", gsync_count);
-        ret = dict_set_dynstr (dict, slv, gf_strdup (slave));
-        if (ret)
-                goto out;
+                sts_val = GF_CALLOC (1, sizeof(gf_gsync_status_t),
+                                     gf_common_mt_gsync_status_t);
+                if (!sts_val) {
+                        gf_log ("", GF_LOG_ERROR, "Out Of Memory");
+                        goto out;
+                }
+
+                /* Creating the brick state file's path */
+                memset(brick_state_file, '\0', PATH_MAX);
+                memcpy (brick_path, brickinfo->path, PATH_MAX - 1);
+                for (i = 0; i < strlen(brick_path) - 1; i++)
+                        if (brick_path[i] == '/')
+                                brick_path[i] = '_';
+                ret = snprintf(brick_state_file, PATH_MAX - 1, "%s%s.status",
+                               georep_session_wrkng_dir, brick_path);
+                brick_state_file[ret] = '\0';
+
+                gf_log ("", GF_LOG_DEBUG, "brick_state_file = %s", brick_state_file);
+
+                memset (tmp, '\0', sizeof(tmp));
+
+                ret = glusterd_gsync_read_frm_status (brick_state_file,
+                                                      tmp, sizeof (tmp));
+                if (ret <= 0) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to read the status"
+                                "file for %s brick for  %s(master), %s(slave) "
+                                "session", brickinfo->path, master, slave);
+                        memcpy (sts_val->slave_node, slave, strlen(slave));
+                        sts_val->slave_node[strlen(slave)] = '\0';
+                        ret = snprintf (sts_val->worker_status, sizeof(sts_val->worker_status), "N/A");
+                        sts_val->worker_status[ret] = '\0';
+                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
+                        sts_val->checkpoint_status[ret] = '\0';
+                        ret = snprintf (sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
+                        sts_val->crawl_status[ret] = '\0';
+                        ret = snprintf (sts_val->files_syncd, sizeof(sts_val->files_syncd), "N/A");
+                        sts_val->files_syncd[ret] = '\0';
+                        ret = snprintf (sts_val->purges_remaining, sizeof(sts_val->purges_remaining), "N/A");
+                        sts_val->purges_remaining[ret] = '\0';
+                        ret = snprintf (sts_val->total_files_skipped, sizeof(sts_val->total_files_skipped), "N/A");
+                        sts_val->total_files_skipped[ret] = '\0';
+                        ret = snprintf (sts_val->files_remaining, sizeof(sts_val->files_remaining), "N/A");
+                        sts_val->files_remaining[ret] = '\0';
+                        ret = snprintf (sts_val->bytes_remaining, sizeof(sts_val->bytes_remaining), "N/A");
+                        sts_val->bytes_remaining[ret] = '\0';
+                        goto store_status;
+                }
+
+                ret = glusterd_gsync_fetch_status_extra (socketfile, sts_val);
+                if (ret || strlen(sts_val->checkpoint_status) == 0) {
+                        gf_log ("", GF_LOG_DEBUG, "No checkpoint status"
+                                "for %s(master), %s(slave)", master, slave);
+                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
+                        sts_val->checkpoint_status[ret] = '\0';
+                }
+
+                ret = glusterd_parse_gsync_status (tmp, sts_val);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to parse the gsync status for %s",
+                                brickinfo->path);
+                        memcpy (sts_val->slave_node, slave, strlen(slave));
+                        sts_val->slave_node[strlen(slave)] = '\0';
+                        ret = snprintf (sts_val->worker_status, sizeof(sts_val->worker_status), "N/A");
+                        sts_val->worker_status[ret] = '\0';
+                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
+                        sts_val->checkpoint_status[ret] = '\0';
+                        ret = snprintf (sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
+                        sts_val->crawl_status[ret] = '\0';
+                        ret = snprintf (sts_val->files_syncd, sizeof(sts_val->files_syncd), "N/A");
+                        sts_val->files_syncd[ret] = '\0';
+                        ret = snprintf (sts_val->purges_remaining, sizeof(sts_val->purges_remaining), "N/A");
+                        sts_val->purges_remaining[ret] = '\0';
+                        ret = snprintf (sts_val->total_files_skipped, sizeof(sts_val->total_files_skipped), "N/A");
+                        sts_val->total_files_skipped[ret] = '\0';
+                        ret = snprintf (sts_val->files_remaining, sizeof(sts_val->files_remaining), "N/A");
+                        sts_val->files_remaining[ret] = '\0';
+                        ret = snprintf (sts_val->bytes_remaining, sizeof(sts_val->bytes_remaining), "N/A");
+                        sts_val->bytes_remaining[ret] = '\0';
+                }
+
+store_status:
+                if ((strcmp (monitor_status, "Stable"))) {
+                        memcpy (sts_val->worker_status, monitor_status, strlen(monitor_status));
+                        sts_val->worker_status[strlen(monitor_status)] = '\0';
+                        ret = snprintf (sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
+                        sts_val->crawl_status[ret] = '\0';
+                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
+                        sts_val->checkpoint_status[ret] = '\0';
+                }
+
+                if (strcmp (sts_val->worker_status, "Active")) {
+                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
+                        sts_val->checkpoint_status[ret] = '\0';
+                        ret = snprintf (sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
+                        sts_val->crawl_status[ret] = '\0';
+                }
+
+                if (!strcmp (sts_val->slave_node, "N/A")) {
+                        memcpy (sts_val->slave_node, slave, strlen(slave));
+                        sts_val->slave_node[strlen(slave)] = '\0';
+                }
+
+                memcpy (sts_val->node, node, strlen(node));
+                sts_val->node[strlen(node)] = '\0';
+                memcpy (sts_val->brick, brickinfo->path, strlen(brickinfo->path));
+                sts_val->brick[strlen(brickinfo->path)] = '\0';
+                memcpy (sts_val->master, master, strlen(master));
+                sts_val->master[strlen(master)] = '\0';
+
+                snprintf (sts_val_name, sizeof (sts_val_name), "status_value%d", gsync_count);
+                ret = dict_set_bin (dict, sts_val_name, sts_val, sizeof(gf_gsync_status_t));
+                if (ret) {
+                        GF_FREE (sts_val);
+                        goto out;
+                }
+
+                gsync_count++;
+                sts_val = NULL;
+        }
 
-        snprintf (sts, sizeof (slv), "status%d", gsync_count);
-        ret = dict_set_dynstr (dict, sts, gf_strdup (buff));
-        if (ret)
-                goto out;
         ret = dict_set_int32 (dict, "gsync-count", gsync_count);
         if (ret)
                 goto out;
 
-        ret = 0;
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
-        return ret;
+out:
+        dict_destroy (confd);
+
+        return 0;
 }
 
 int
 glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
-                                      dict_t *resp_dict)
+                                      dict_t *resp_dict, char *path_list,
+                                      char *conf_path, gf_boolean_t is_force)
 {
 
         int                    ret = 0;
-        uuid_t                 uuid = {0, };
         glusterd_conf_t        *priv = NULL;
         char                   *status_msg = NULL;
+        gf_boolean_t           is_running = _gf_false;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (slave);
@@ -1366,26 +3225,30 @@ glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
 
         priv = THIS->private;
 
-        if (glusterd_gsync_get_uuid (slave, volinfo, uuid))
-                /* session does not exist, nothing to do */
+        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (!ret && (_gf_true != is_running))
+                /* gsynd not running, nothing to do */
                 goto out;
-        if (uuid_compare (priv->uuid, uuid) == 0) {
-                ret = stop_gsync (volinfo->volname, slave, &status_msg);
-                if (ret == 0 && status_msg)
-                        ret = dict_set_str (resp_dict, "gsync-status",
-                                            status_msg);
-                if (ret == 0)
-                        ret = glusterd_start_gsync (volinfo, slave,
-                                                    uuid_utoa(priv->uuid), NULL);
-        }
+
+        ret = stop_gsync (volinfo->volname, slave, &status_msg,
+                          conf_path, is_force);
+        if (ret == 0 && status_msg)
+                ret = dict_set_str (resp_dict, "gsync-status",
+                                    status_msg);
+        if (ret == 0)
+                ret = glusterd_start_gsync (volinfo, slave, path_list,
+                                            conf_path, uuid_utoa(MY_UUID),
+                                            NULL);
 
  out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int32_t
-glusterd_marker_create_volfile (glusterd_volinfo_t *volinfo)
+static int32_t
+glusterd_marker_changelog_create_volfile (glusterd_volinfo_t *volinfo)
 {
         int32_t          ret     = 0;
 
@@ -1408,58 +3271,83 @@ out:
         return ret;
 }
 
-int
-glusterd_set_marker_gsync (glusterd_volinfo_t *volinfo)
+static int
+glusterd_set_gsync_knob (glusterd_volinfo_t *volinfo, char *key, int *vc)
 {
-        int                      ret     = -1;
-        int                      marker_set = _gf_false;
-        char                    *gsync_status = NULL;
+        int   ret          = -1;
+        int   conf_enabled = _gf_false;
+        char *knob_on      = NULL;
 
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
 
-        marker_set = glusterd_volinfo_get_boolean (volinfo, VKEY_MARKER_XTIME);
-        if (marker_set == -1) {
-                gf_log ("", GF_LOG_ERROR, "failed to get the marker status");
-                ret = -1;
+        conf_enabled = glusterd_volinfo_get_boolean (volinfo, key);
+        if (conf_enabled == -1) {
+                gf_log ("", GF_LOG_ERROR,
+                        "failed to get key %s from volinfo", key);
                 goto out;
         }
 
-        if (marker_set == _gf_false) {
-                gsync_status = gf_strdup ("on");
-                if (gsync_status == NULL) {
+        ret = 0;
+        if (conf_enabled == _gf_false) {
+                *vc = 1;
+                knob_on = gf_strdup ("on");
+                if (knob_on == NULL) {
                         ret = -1;
                         goto out;
                 }
 
                 ret = glusterd_gsync_volinfo_dict_set (volinfo,
-                                                       VKEY_MARKER_XTIME, gsync_status);
-                if (ret < 0)
-                        goto out;
-
-                ret = glusterd_marker_create_volfile (volinfo);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Setting dict failed");
-                        goto out;
-                }
+                                                       key, knob_on);
         }
-        ret = 0;
 
-out:
+ out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+static int
+glusterd_set_gsync_confs (glusterd_volinfo_t *volinfo)
+{
+        int ret             = -1;
+        int volfile_changed = 0;
+
+        ret = glusterd_set_gsync_knob (volinfo,
+                                       VKEY_MARKER_XTIME, &volfile_changed);
+        if (ret)
+                goto out;
+
+        /**
+         * enable ignore-pid-check blindly as it could be needed for
+         * cascading setups.
+         */
+        ret = glusterd_set_gsync_knob (volinfo, VKEY_MARKER_XTIME_FORCE,
+                                       &volfile_changed);
+        if (ret)
+                goto out;
 
+        ret = glusterd_set_gsync_knob (volinfo,
+                                       VKEY_CHANGELOG, &volfile_changed);
+        if (ret)
+                goto out;
 
+        if (volfile_changed)
+                ret = glusterd_marker_changelog_create_volfile (volinfo);
 
-int
-glusterd_get_gsync_status_mst_slv( glusterd_volinfo_t *volinfo,
-                                   char *slave, dict_t *rsp_dict)
+ out:
+        return ret;
+}
+
+static int
+glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
+                                   char *slave, char *conf_path,
+                                   dict_t *rsp_dict, char *node)
 {
+        char              *statefile = NULL;
         uuid_t             uuid = {0, };
         glusterd_conf_t    *priv = NULL;
         int                ret = 0;
+        struct stat        stbuf = {0, };
 
         GF_ASSERT (volinfo);
         GF_ASSERT (slave);
@@ -1469,24 +3357,45 @@ glusterd_get_gsync_status_mst_slv( glusterd_volinfo_t *volinfo,
         priv = THIS->private;
 
         ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if ((ret == 0) && (uuid_compare (priv->uuid, uuid) != 0))
-                goto out;
-
         if (ret) {
-                ret = 0;
                 gf_log ("", GF_LOG_INFO, "geo-replication status %s %s :"
                         "session is not active", volinfo->volname, slave);
-                goto out;
+
+                ret = glusterd_get_statefile_name (volinfo, slave,
+                                                   conf_path, &statefile);
+                if (ret) {
+                        if (!strstr(slave, "::"))
+                                gf_log ("", GF_LOG_INFO,
+                                        "%s is not a valid slave url.", slave);
+                        else
+                                gf_log ("", GF_LOG_INFO, "Unable to get"
+                                        " statefile's name");
+                        ret = 0;
+                        goto out;
+                }
+
+                ret = lstat (statefile, &stbuf);
+                if (ret) {
+                        gf_log ("", GF_LOG_INFO, "%s statefile not present.",
+                                statefile);
+                        ret = 0;
+                        goto out;
+                }
         }
 
-        ret = glusterd_read_status_file (volinfo->volname, slave, rsp_dict);
- out:
+        ret = glusterd_read_status_file (volinfo, slave, conf_path,
+                                         rsp_dict, node);
+out:
+        if (statefile)
+                GF_FREE (statefile);
+
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
 static int
-glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict)
+glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
+                               char *node)
 {
         glusterd_gsync_status_temp_t  param = {0, };
 
@@ -1494,13 +3403,14 @@ glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict)
 
         param.rsp_dict = rsp_dict;
         param.volinfo = volinfo;
+        param.node = node;
         dict_foreach (volinfo->gsync_slaves, _get_status_mst_slv, &param);
 
         return 0;
 }
 
 static int
-glusterd_get_gsync_status_all ( dict_t *rsp_dict)
+glusterd_get_gsync_status_all (dict_t *rsp_dict, char *node)
 {
 
         int32_t                 ret = 0;
@@ -1513,7 +3423,7 @@ glusterd_get_gsync_status_all ( dict_t *rsp_dict)
         GF_ASSERT (priv);
 
         list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict);
+                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict, node);
                 if (ret)
                         goto out;
         }
@@ -1529,15 +3439,22 @@ glusterd_get_gsync_status (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
         char               *slave  = NULL;
         char               *volname = NULL;
+        char               *conf_path = NULL;
         char               errmsg[PATH_MAX] = {0, };
         gf_boolean_t       exists = _gf_false;
         glusterd_volinfo_t *volinfo = NULL;
         int                ret = 0;
+        char my_hostname[256] = {0,};
 
+        ret = gethostname(my_hostname, 256);
+        if (ret) {
+                /* stick to N/A */
+                (void) strcpy (my_hostname, "N/A");
+        }
 
         ret = dict_get_str (dict, "master", &volname);
         if (ret < 0){
-                ret = glusterd_get_gsync_status_all (rsp_dict);
+                ret = glusterd_get_gsync_status_all (rsp_dict, my_hostname);
                 goto out;
         }
 
@@ -1555,317 +3472,452 @@ glusterd_get_gsync_status (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 
         ret = dict_get_str (dict, "slave", &slave);
         if (ret < 0) {
-                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict);
+                ret = glusterd_get_gsync_status_mst (volinfo,
+                                                     rsp_dict, my_hostname);
                 goto out;
         }
 
-        ret = glusterd_get_gsync_status_mst_slv (volinfo, slave, rsp_dict);
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
+        ret = glusterd_get_gsync_status_mst_slv (volinfo, slave, conf_path,
+                                                 rsp_dict, my_hostname);
 
  out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int
-glusterd_send_sigstop (pid_t pid)
-{
-        int ret = 0;
-        ret = kill (pid, SIGSTOP);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, GEOREP"failed to send SIGSTOP signal");
-        return ret;
-}
-
-int
-glusterd_send_sigcont (pid_t pid)
-{
-        int ret = 0;
-        ret = kill (pid, SIGCONT);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, GEOREP"failed to send SIGCONT signal");
-        return ret;
-}
-
-/*
- * Log rotations flow is something like this:
- * - Send SIGSTOP to process group (this will stop monitor/worker process
- *   and also the slave if it's local)
- * - Rotate log file for monitor/worker
- * - Rotate log file for slave if it's local
- * - Send SIGCONT to the process group. Monitor wakes up, kills the worker
- *   (this is done in the SIGCONT handler), which results in the termination
- *   of the slave (local/remote). After returning from signal handler,
- *   monitor detects absence of worker and starts it again, which in-turn
- *   starts the slave.
- */
-int
-glusterd_send_log_rotate_signal (pid_t pid, char *logfile1, char *logfile2)
+static int
+glusterd_gsync_delete (glusterd_volinfo_t *volinfo, char *slave, char *slave_ip,
+                       char *slave_vol, char *path_list, dict_t *dict,
+                       dict_t *resp_dict, char **op_errstr)
 {
-        int         ret         = 0;
-        struct stat stbuf       = {0,};
-        char rlogfile[PATH_MAX] = {0,};
-        time_t      rottime     = 0;
-
-        ret = glusterd_send_sigstop (-pid);
-        rottime = time (NULL);
+        int32_t         ret     = -1;
+        runner_t        runner    = {0,};
+        glusterd_conf_t *priv   = NULL;
+        char            *master = NULL;
+        char            *gl_workdir = NULL;
+        char             geo_rep_dir[PATH_MAX] = "";
+        char            *conf_path = NULL;
 
-        snprintf (rlogfile, sizeof (rlogfile), "%s.%"PRIu64, logfile1,
-                  (uint64_t) rottime);
-        ret = rename (logfile1, rlogfile);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "rename failed for geo-rep log file");
+        GF_ASSERT (slave);
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (slave_vol);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (dict);
+        GF_ASSERT (resp_dict);
 
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
 
-        snprintf (rlogfile, sizeof (rlogfile), "%s.%"PRIu64, logfile2,
-                  (uint64_t) rottime);
-        ret = stat (logfile2, &stbuf);
+        ret = dict_get_str (dict, "conf_path", &conf_path);
         if (ret) {
-                if (errno != ENOENT)
-                        gf_log("", GF_LOG_ERROR, "stat failed for slave log"
-                               " file: %s", logfile2);
-                else {
-                        gf_log ("", GF_LOG_DEBUG, "Slave is not local, skipping rotation");
-                        ret = 0;
-                }
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
                 goto out;
         }
 
-        ret = rename (logfile2, rlogfile);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "rename failed for geo-rep slave log file");
-
- out:
-        ret = glusterd_send_sigcont (-pid);
-
-        return ret;
-}
+        gl_workdir = priv->workdir;
+        master = "";
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          "--delete", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
 
-int
-glusterd_get_pid_from_file (char *master, char *slave, pid_t *pid)
-{
-        int ret                = -1;
-        int pfd                = 0;
-        char pidfile[PATH_MAX] = {0,};
-        char buff[1024]        = {0,};
+        if (volinfo) {
+                master = volinfo->volname;
+                runner_argprintf (&runner, ":%s", master);
+        }
+        runner_add_arg (&runner, slave);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "gsyncd failed to "
+                        "delete session info for %s and %s peers",
+                        master, slave);
 
-        pfd = gsyncd_getpidfile (master, slave, pidfile);
+                gf_asprintf (op_errstr, "gsyncd failed to "
+                             "delete session info for %s and %s peers",
+                             master, slave);
 
-        if (pfd == -2) {
-                gf_log ("", GF_LOG_ERROR, GEOREP" log-rotate validation "
-                        " failed for %s & %s", master, slave);
-                goto out;
-        }
-        if (gsync_status_byfd (pfd) == -1) {
-                gf_log ("", GF_LOG_ERROR, "gsyncd b/w %s & %s is not"
-                        " running", master, slave);
                 goto out;
         }
 
-        ret = read (pfd, buff, 1024);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, GEOREP" cannot read pid from pid-file");
-                goto out;
-        }
+        ret = snprintf (geo_rep_dir, sizeof(geo_rep_dir) - 1,
+                        "%s/"GEOREP"/%s_%s_%s", gl_workdir,
+                        volinfo->volname, slave_ip, slave_vol);
+        geo_rep_dir[ret] = '\0';
 
-        close(pfd);
+        ret = rmdir (geo_rep_dir);
+        if (ret) {
+                if (errno == ENOENT)
+                        gf_log ("", GF_LOG_DEBUG, "Geo Rep Dir(%s) Not Present.",
+                                geo_rep_dir);
+                else {
+                        gf_log ("", GF_LOG_ERROR, "Unable to delete "
+                                "Geo Rep Dir(%s). Error: %s", geo_rep_dir,
+                                strerror (errno));
+                        goto out;
+                }
+        }
 
-        *pid = strtol (buff, NULL, 10);
         ret = 0;
 
- out:
+        gf_asprintf (op_errstr, "delete successful");
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd_do_gsync_log_rotate (char *master, char *slave, uuid_t *uuid, char **op_errstr)
+glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int              ret     = 0;
-        glusterd_conf_t *priv    = NULL;
-        pid_t            pid     = 0;
-        char log_file1[PATH_MAX] = {0,};
-        char log_file2[PATH_MAX] = {0,};
-
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
+        char                   buf[PATH_MAX]          = "";
+        char                   cmd_arg_name[PATH_MAX] = "";
+        char                   output_name[PATH_MAX]  = "";
+        char                   errmsg[PATH_MAX]       = "";
+        char                  *ptr                    = NULL;
+        char                  *bufp                   = NULL;
+        char                  *command                = NULL;
+        char                 **cmd_args               = NULL;
+        int                    ret                    = -1;
+        int                    i                      = -1;
+        int                    cmd_args_count         = 0;
+        int                    output_count           = 0;
+        glusterd_conf_t       *priv                   = NULL;
+        runner_t               runner                 = {0,};
 
-        priv = THIS->private;
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
 
-        ret = glusterd_get_pid_from_file (master, slave, &pid);
-        if (ret)
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
                 goto out;
+        }
 
-        /* log file */
-        ret = glusterd_gsyncd_getlogfile (master, slave, log_file1);
-        if (ret)
+        ret = dict_get_str (dict, "command", &command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get command from dict");
                 goto out;
+        }
 
-        /* slave log file */
-        ret = glusterd_gsync_get_slave_log_file (master, slave, log_file2);
+        ret = dict_get_int32 (dict, "cmd_args_count", &cmd_args_count);
         if (ret)
-                goto out;
-
-        ret = glusterd_send_log_rotate_signal (pid, log_file1, log_file2);
-
- out:
-        if (ret && op_errstr)
-                *op_errstr = gf_strdup("Error rotating log file");
-        return ret;
-}
-
-int
-glusterd_do_gsync_log_rotation_mst_slv (glusterd_volinfo_t *volinfo, char *slave,
-                                        char **op_errstr)
-{
-        uuid_t           uuid = {0, };
-        glusterd_conf_t *priv = NULL;
-        int              ret  = 0;
-        char             errmsg[1024] = {0,};
-        xlator_t        *this    = NULL;
+                gf_log ("", GF_LOG_INFO, "No cmd_args_count");
+
+        if (cmd_args_count) {
+                cmd_args = GF_CALLOC (cmd_args_count, sizeof (char*),
+                                      gf_common_mt_char);
+                if (!cmd_args) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to calloc. "
+                                "Errno = %s", strerror(errno));
+                        goto out;
+                }
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (THIS);
-        this = THIS;
-        GF_ASSERT (this->private);
-        priv = this->private;
+                for (i=1; i <= cmd_args_count; i++) {
+                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
+                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
+                                  "cmd_arg_%d", i);
+                        ret = dict_get_str (dict, cmd_arg_name, &cmd_args[i-1]);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s in dict",
+                                        cmd_arg_name);
+                                goto out;
+                        }
+                }
+        }
 
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if ((ret == 0) && (uuid_compare (priv->uuid, uuid) != 0))
+        runinit (&runner);
+        runner_argprintf (&runner, GSYNCD_PREFIX"/peer_%s", command);
+        for (i=0; i < cmd_args_count; i++)
+                runner_add_arg (&runner, cmd_args[i]);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to "
+                          "execute command. Error : %s",
+                          strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                synclock_lock (&priv->big_lock);
                 goto out;
+        }
+
+        do {
+                ptr = fgets(buf, sizeof(buf), runner_chio (&runner, STDOUT_FILENO));
+                if (ptr) {
+                        ret = dict_get_int32 (rsp_dict, "output_count", &output_count);
+                        if (ret)
+                                output_count = 1;
+                        else
+                                output_count++;
+                        memset (output_name, '\0', sizeof (output_name));
+                        snprintf (output_name, sizeof (output_name),
+                                  "output_%d", output_count);
+                        if (buf[strlen(buf) - 1] == '\n')
+                                buf[strlen(buf) - 1] = '\0';
+                        bufp = gf_strdup (buf);
+                        if (!bufp)
+                                gf_log ("", GF_LOG_ERROR, "gf_strdup failed.");
+                        ret = dict_set_dynstr (rsp_dict, output_name, bufp);
+                        if (ret) {
+                                GF_FREE (bufp);
+                                gf_log ("", GF_LOG_ERROR, "output set failed.");
+                        }
+                        ret = dict_set_int32 (rsp_dict, "output_count", output_count);
+                        if (ret)
+                                gf_log ("", GF_LOG_ERROR, "output_count set failed.");
+                }
+        } while (ptr);
 
+        ret = runner_end (&runner);
         if (ret) {
-                snprintf(errmsg, sizeof(errmsg), "geo-replication session b/w %s %s not active",
-                         volinfo->volname, slave);
-                gf_log (this->name, GF_LOG_WARNING, "%s", errmsg);
-                if (op_errstr)
-                        *op_errstr = gf_strdup(errmsg);
+                snprintf (errmsg, sizeof (errmsg), "Unable to "
+                          "end. Error : %s",
+                          strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                synclock_lock (&priv->big_lock);
                 goto out;
         }
+        synclock_lock (&priv->big_lock);
 
-        ret = glusterd_do_gsync_log_rotate (volinfo->volname, slave, &uuid, op_errstr);
+        ret = 0;
+out:
+        if (cmd_args) {
+                GF_FREE (cmd_args);
+                cmd_args = NULL;
+        }
 
- out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-static void
-_iterate_log_rotate_mst_slv (dict_t *this, char *key, data_t *value, void *data)
+int
+glusterd_op_copy_file (dict_t *dict, char **op_errstr)
 {
-        glusterd_gsync_status_temp_t  *param = NULL;
-        char                          *slave = NULL;
+        char             abs_filename[PATH_MAX] = "";
+        char             errmsg[PATH_MAX]       = "";
+        char            *filename               = NULL;
+        char            *host_uuid              = NULL;
+        char             uuid_str [64]          = {0};
+        char            *contents               = NULL;
+        char             buf[1024]              = "";
+        int              ret                    = -1;
+        int              fd                     = -1;
+        int              bytes_writen           = 0;
+        int              bytes_read             = 0;
+        int              contents_size          = -1;
+        int              file_mode              = -1;
+        glusterd_conf_t *priv                   = NULL;
+        struct stat      stbuf                  = {0,};
 
-        param = (glusterd_gsync_status_temp_t *) data;
 
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
 
-        slave = strchr (value->data, ':');
-        if (slave)
-                slave++;
-        else {
-                gf_log ("", GF_LOG_ERROR, "geo-replication log-rotate: slave (%s) "
-                        "not conforming to format", slave);
-                return;
+        ret = dict_get_str (dict, "host-uuid", &host_uuid);
+        if (ret < 0)
+                goto out;
+
+        ret = dict_get_str (dict, "source", &filename);
+        if (ret < 0) {
+               gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                       " filename from dict.");
+               *op_errstr = gf_strdup ("command unsuccessful");
+               goto out;
         }
+        snprintf (abs_filename, sizeof(abs_filename),
+                  "%s/%s", priv->workdir, filename);
 
-        (void) glusterd_do_gsync_log_rotation_mst_slv (param->volinfo, slave, NULL);
-}
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = lstat (abs_filename, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                 " does not exist in %s", priv->workdir);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-int
-glusterd_do_gsync_log_rotation_mst (glusterd_volinfo_t *volinfo)
-{
-        glusterd_gsync_status_temp_t  param = {0, };
+                contents = GF_CALLOC(1, stbuf.st_size+1, gf_common_mt_char);
+                if (!contents) {
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to allocate memory");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        GF_ASSERT (volinfo);
+                fd = open (abs_filename, O_RDONLY);
+                if (fd < 0) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
+                                  abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        param.volinfo = volinfo;
-        dict_foreach (volinfo->gsync_slaves, _iterate_log_rotate_mst_slv, &param);
-        return 0;
-}
+                do {
+                        ret = read (fd, buf, sizeof(buf));
+                        if (ret > 0) {
+                                memcpy (contents+bytes_read, buf, ret);
+                                bytes_read += ret;
+                                memset (buf, '\0', sizeof(buf));
+                        }
+                } while (ret > 0);
 
-static int
-glusterd_rotate_gsync_all ()
-{
-        int32_t             ret     = 0;
-        glusterd_conf_t    *priv    = NULL;
-        glusterd_volinfo_t *volinfo = NULL;
+                if (bytes_read != stbuf.st_size) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to read all "
+                                  "the data from %s", abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        GF_ASSERT (THIS);
-        priv = THIS->private;
+                ret = dict_set_int32 (dict, "contents_size", stbuf.st_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " contents size in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-        GF_ASSERT (priv);
+                ret = dict_set_int32 (dict, "file_mode",
+                                      (int32_t)stbuf.st_mode);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " file mode in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                ret = glusterd_do_gsync_log_rotation_mst (volinfo);
-                if (ret)
+                ret = dict_set_bin (dict, "common_pem_contents",
+                                    contents, stbuf.st_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " pem contents in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
                         goto out;
-        }
+                }
+                close (fd);
+        } else {
+                ret = dict_get_bin (dict, "common_pem_contents",
+                                    (void **) &contents);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
+                                  " pem contents in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
+                ret = dict_get_int32 (dict, "contents_size", &contents_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " contents size in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-static int
-glusterd_rotate_gsync_logs (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
-{
-        char               *slave   = NULL;
-        char               *volname = NULL;
-        char errmsg[1024]           = {0,};
-        gf_boolean_t        exists  = _gf_false;
-        glusterd_volinfo_t *volinfo = NULL;
-        int                 ret     = 0;
+                ret = dict_get_int32 (dict, "file_mode", &file_mode);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
+                                  " file mode in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
 
-        ret = dict_get_str (dict, "master", &volname);
-        if (ret < 0) {
-                ret = glusterd_rotate_gsync_all ();
-                goto out;
-        }
+                fd = open (abs_filename, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+                if (fd < 0) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
+                                  abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                snprintf (errmsg, sizeof(errmsg), "Volume %s does not"
-                          " exist", volname);
-                gf_log ("", GF_LOG_WARNING, "%s", errmsg);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
+                bytes_writen = write (fd, contents, contents_size);
 
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0) {
-                ret = glusterd_do_gsync_log_rotation_mst (volinfo);
-                goto out;
-        }
+                if (bytes_writen != contents_size) {
+                        snprintf (errmsg, sizeof (errmsg), "Failed to write"
+                                  " to %s", abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
 
-        ret = glusterd_do_gsync_log_rotation_mst_slv (volinfo, slave, op_errstr);
+                fchmod (fd, file_mode);
+                close (fd);
+        }
 
- out:
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-
 int
 glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
         int32_t             ret     = -1;
         int32_t             type    = -1;
-        dict_t             *ctx    = NULL;
-        dict_t             *resp_dict = NULL;
         char               *host_uuid = NULL;
         char               *slave  = NULL;
+        char               *slave_ip  = NULL;
+        char               *slave_vol = NULL;
         char               *volname = NULL;
+        char               *path_list = NULL;
         glusterd_volinfo_t *volinfo = NULL;
         glusterd_conf_t    *priv = NULL;
+        gf_boolean_t        is_force = _gf_false;
         char               *status_msg = NULL;
-        uuid_t              uuid = {0, };
+        gf_boolean_t        is_running = _gf_false;
+        char               *conf_path = NULL;
 
         GF_ASSERT (THIS);
         GF_ASSERT (THIS->private);
         GF_ASSERT (dict);
         GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
 
         priv = THIS->private;
 
@@ -1877,37 +3929,67 @@ glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         if (ret < 0)
                 goto out;
 
-        ctx = glusterd_op_get_ctx ();
-        resp_dict = ctx ? ctx : rsp_dict;
-        GF_ASSERT (resp_dict);
-
         if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
-                ret = glusterd_get_gsync_status (dict, op_errstr, resp_dict);
+                ret = glusterd_get_gsync_status (dict, op_errstr, rsp_dict);
                 goto out;
         }
 
-        if (type == GF_GSYNC_OPTION_TYPE_ROTATE) {
-                ret = glusterd_rotate_gsync_logs (dict, op_errstr, resp_dict);
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret < 0)
+                goto out;
+
+        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave volume name.");
                 goto out;
+        }
 
+        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave volume name.");
+                goto out;
         }
 
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0)
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
                 goto out;
+        }
 
         if (dict_get_str (dict, "master", &volname) == 0) {
                 ret = glusterd_volinfo_find (volname, &volinfo);
                 if (ret) {
-                        gf_log ("", GF_LOG_WARNING, "Volinfo for %s (master) not found",
-                                volname);
+                        gf_log ("", GF_LOG_WARNING, "Volinfo for %s (master)"
+                                " not found", volname);
                         goto out;
                 }
+
+                ret = glusterd_get_local_brickpaths (volinfo, &path_list);
         }
 
         if (type == GF_GSYNC_OPTION_TYPE_CONFIG) {
-                ret = glusterd_gsync_configure (volinfo, slave, dict, resp_dict,
-                                                op_errstr);
+                ret = glusterd_gsync_configure (volinfo, slave, path_list,
+                                                dict, rsp_dict, op_errstr);
+                if (!ret) {
+                        ret = dict_set_str (rsp_dict, "conf_path", conf_path);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to store conf_file_path.");
+                                goto out;
+                        }
+                }
+                goto out;
+        }
+
+        if (type == GF_GSYNC_OPTION_TYPE_DELETE) {
+                ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
+                if (ret && !is_force && path_list)
+                        goto out;
+
+                ret = glusterd_gsync_delete (volinfo, slave, slave_ip,
+                                             slave_vol, path_list, dict,
+                                             rsp_dict, op_errstr);
                 goto out;
         }
 
@@ -1916,48 +3998,671 @@ glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                 goto out;
         }
 
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
         if (type == GF_GSYNC_OPTION_TYPE_START) {
 
-                ret = glusterd_set_marker_gsync (volinfo);
+                ret = glusterd_set_gsync_confs (volinfo);
                 if (ret != 0) {
-                        gf_log ("", GF_LOG_WARNING, "marker start failed");
-                        *op_errstr = gf_strdup ("failed to initialize indexing");
+                        gf_log ("", GF_LOG_WARNING, "marker/changelog"
+                                " start failed");
+                        *op_errstr = gf_strdup ("Index initialization failed");
                         ret = -1;
                         goto out;
                 }
-                ret = glusterd_store_slave_in_info(volinfo, slave,
-                                                   host_uuid, op_errstr);
-                if (ret)
-                        goto out;
 
-                ret = glusterd_start_gsync (volinfo, slave, host_uuid,
-                                            op_errstr);
+                ret = glusterd_start_gsync (volinfo, slave, path_list,
+                                            conf_path, host_uuid, op_errstr);
         }
 
         if (type == GF_GSYNC_OPTION_TYPE_STOP) {
-
-                ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-                if (ret) {
+                ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                          slave, conf_path,
+                                                          &is_running);
+                if (!ret && !is_force && path_list &&
+                    (_gf_true != is_running)) {
                         gf_log ("", GF_LOG_WARNING, GEOREP" is not set up for"
                                 "%s(master) and %s(slave)", volname, slave);
                         *op_errstr = strdup (GEOREP" is not set up");
                         goto out;
                 }
 
-                ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
-                if (ret)
+                ret = stop_gsync (volname, slave, &status_msg,
+                                  conf_path, is_force);
+                if (ret == 0 && status_msg)
+                        ret = dict_set_str (rsp_dict, "gsync-status",
+                                            status_msg);
+                if (ret != 0 && !is_force && path_list)
+                        *op_errstr = gf_strdup ("internal error");
+
+                if (!ret) {
+                        ret = glusterd_create_status_file (volinfo->volname,
+                                                           slave, slave_ip,
+                                                           slave_vol,"Stopped");
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to update"
+                                        "state_file. Error : %s",
+                                        strerror (errno));
+                        }
+                }
+        }
+
+out:
+        if (path_list) {
+                GF_FREE (path_list);
+                path_list = NULL;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char **slave_ip, char **slave_vol,
+                                     char **conf_path, char **op_errstr)
+{
+        int                ret                = -1;
+        char               confpath[PATH_MAX] = "";
+        glusterd_conf_t   *priv               = NULL;
+        char              *slave              = NULL;
+
+        GF_ASSERT (THIS);
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret || !slave) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave from dict");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_slave_info (slave, slave_ip, slave_vol, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "slave_ip", *slave_ip);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave IP.");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "slave_vol", *slave_vol);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave volume name.");
+                goto out;
+        }
+
+        ret = snprintf (confpath, sizeof(confpath) - 1,
+                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                        priv->workdir, volinfo->volname,
+                        *slave_ip, *slave_vol);
+        confpath[ret] = '\0';
+        *conf_path = gf_strdup (confpath);
+        if (!(*conf_path)) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to gf_strdup. Error: %s", strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "conf_path", *conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store conf_path");
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
+        return ret;
+
+}
+
+int
+glusterd_get_slave_info (char *slave, char **slave_ip,
+                         char **slave_vol, char **op_errstr)
+{
+        char     *tmp              = NULL;
+        char     *save_ptr         = NULL;
+        char    **linearr          = NULL;
+        int32_t   ret              = -1;
+        char      errmsg[PATH_MAX] = "";
+
+        ret = glusterd_urltransform_single (slave, "normalize",
+                                            &linearr);
+        if (ret == -1) {
+                ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                "Invalid Url: %s", slave);
+                errmsg[ret] = '\0';
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "Failed to normalize url");
+                goto out;
+        }
+
+        tmp = strtok_r (linearr[0], "/", &save_ptr);
+        tmp = strtok_r (NULL, "/", &save_ptr);
+        slave = strtok_r (tmp, ":", &save_ptr);
+        if (slave) {
+                ret = glusterd_mountbroker_check (&slave, op_errstr);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Invalid slave url: %s", *op_errstr);
                         goto out;
+                }
 
-                if (uuid_compare (priv->uuid, uuid) != 0) {
+                *slave_ip = gf_strdup (slave);
+                if (!*slave_ip) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
                         goto out;
                 }
+                gf_log ("", GF_LOG_DEBUG, "Slave IP : %s", *slave_ip);
+                ret = 0;
+        } else {
+                gf_log ("", GF_LOG_ERROR, "Invalid slave name");
+                goto out;
+        }
 
-                ret = stop_gsync (volname, slave, &status_msg);
-                if (ret == 0 && status_msg)
-                        ret = dict_set_str (resp_dict, "gsync-status",
-                                            status_msg);
-                if (ret != 0)
-                        *op_errstr = gf_strdup ("internal error");
+        slave = strtok_r (NULL, ":", &save_ptr);
+        if (slave) {
+                *slave_vol = gf_strdup (slave);
+                if (!*slave_vol) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
+                        goto out;
+                }
+                gf_log ("", GF_LOG_DEBUG, "Slave Vol : %s", *slave_vol);
+                ret = 0;
+        } else {
+                gf_log ("", GF_LOG_ERROR, "Invalid slave name");
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static void
+runinit_gsyncd_setrx (runner_t *runner, char *conf_path)
+{
+        runinit (runner);
+        runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (runner, "%s", conf_path);
+        runner_add_arg (runner, "--config-set-rx");
+}
+
+static int
+glusterd_check_gsync_present (int *valid_state)
+{
+        char                buff[PATH_MAX] = {0, };
+        runner_t            runner = {0,};
+        char               *ptr = NULL;
+        int                 ret = 0;
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                if (errno == ENOENT) {
+                        gf_log ("glusterd", GF_LOG_INFO, GEOREP
+                                 " module not installed in the system");
+                        *valid_state = 0;
+                }
+                else {
+                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP
+                                  " module not working as desired");
+                        *valid_state = -1;
+                }
+                goto out;
+        }
+
+        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
+        if (ptr) {
+                if (!strstr (buff, "gsyncd")) {
+                        ret = -1;
+                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
+                                 "working as desired");
+                        *valid_state = -1;
+                        goto out;
+                }
+        } else {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
+                         "working as desired");
+                *valid_state = -1;
+                goto out;
+        }
+
+        ret = 0;
+ out:
+
+        runner_end (&runner);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+create_conf_file (glusterd_conf_t *conf, char *conf_path)
+#define RUN_GSYNCD_CMD do {                                                          \
+        ret = runner_run_reuse (&runner);                                            \
+        if (ret == -1) {                                                             \
+                runner_log (&runner, "glusterd", GF_LOG_ERROR, "command failed");    \
+                runner_end (&runner);                                                \
+                goto out;                                                            \
+        }                                                                            \
+        runner_end (&runner);                                                        \
+} while (0)
+{
+        int ret = 0;
+        runner_t runner = {0,};
+        char georepdir[PATH_MAX] = {0,};
+        int valid_state = 0;
+
+        valid_state = -1;
+        ret = glusterd_check_gsync_present (&valid_state);
+        if (-1 == ret) {
+                ret = valid_state;
+                goto out;
+        }
+
+        ret = snprintf (georepdir, sizeof(georepdir) - 1, "%s/"GEOREP,
+                        conf->workdir);
+        georepdir[ret] = '\0';
+
+        /************
+         * master pre-configuration
+         ************/
+
+        /* remote-gsyncd */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "remote-gsyncd", GSYNCD_PREFIX"/gsyncd", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
+                         ".", "^ssh:", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-params",
+                         "aux-gfid-mount",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ssh-command */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "ssh-command");
+        runner_argprintf (&runner,
+                          "ssh -oPasswordAuthentication=no "
+                           "-oStrictHostKeyChecking=no "
+                           "-i %s/secret.pem", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ssh-command tar */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "ssh-command-tar");
+        runner_argprintf (&runner,
+                          "ssh -oPasswordAuthentication=no "
+                           "-oStrictHostKeyChecking=no "
+                           "-i %s/tar_ssh.pem", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* pid-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "pid-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* geo-rep-working-dir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "georep-session-working-dir");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-socket */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-socket-unencoded");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.socket", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* socketdir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "gluster-log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ignore-deletes */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* special-sync-mode */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* change-detector == changelog */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg(&runner, "working-dir");
+        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                         DEFAULT_VAR_RUN_DIRECTORY);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /************
+         * slave pre-configuration
+         ************/
+
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-params",
+                         "aux-gfid-mount",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* MountBroker log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file-mbr",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "gluster-log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+ out:
+        return ret ? -1 : 0;
+}
+
+static int
+glusterd_create_essential_dir_files (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char *slave, char *slave_ip,
+                                     char *slave_vol, char **op_errstr)
+{
+        int                ret              = -1;
+        char              *conf_path        = NULL;
+        char              *statefile        = NULL;
+        char               buf[PATH_MAX]    = "";
+        char               errmsg[PATH_MAX] = "";
+        glusterd_conf_t   *conf             = NULL;
+        struct stat        stbuf            = {0,};
+
+        GF_ASSERT (THIS);
+        conf = THIS->private;
+
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch conf file path.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "statefile", &statefile);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch statefile path.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = snprintf (buf, sizeof(buf) - 1, "%s/"GEOREP"/%s_%s_%s",
+                        conf->workdir, volinfo->volname, slave_ip, slave_vol);
+        buf[ret] = '\0';
+        ret = mkdir_p (buf, 0777, _gf_true);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                          ". Error : %s", buf, strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s",
+                        volinfo->volname);
+        buf[ret] = '\0';
+        ret = mkdir_p (buf, 0777, _gf_true);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                          ". Error : %s", buf, strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Session already running."
+                        " Not creating config file again.");
+        } else {
+                ret = create_conf_file (conf, conf_path);
+                if (ret || lstat (conf_path, &stbuf)) {
+                        snprintf (errmsg, sizeof (errmsg), "Failed to create"
+                                  " config file(%s).", conf_path);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+        }
+
+        ret = lstat (statefile, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Session already running."
+                        " Not creating status file again.");
+                goto out;
+        } else {
+                ret = glusterd_create_status_file (volinfo->volname, slave,
+                                                   slave_ip, slave_vol,
+                                                   "Not Started");
+                if (ret || lstat (statefile, &stbuf)) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                                  ". Error : %s", statefile, strerror (errno));
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        char                common_pem_file[PATH_MAX] = "";
+        char                errmsg[PATH_MAX]          = "";
+        char                hooks_args[PATH_MAX]      = "";
+        char                uuid_str [64]             = "";
+        char               *host_uuid                 = NULL;
+        char               *slave_ip                  = NULL;
+        char               *slave_vol                 = NULL;
+        char               *arg_buf                   = NULL;
+        char               *volname                   = NULL;
+        char               *slave                     = NULL;
+        int32_t             ret                       = -1;
+        int32_t             is_pem_push               = -1;
+        gf_boolean_t        is_force                  = -1;
+        glusterd_conf_t    *conf                      = NULL;
+        glusterd_volinfo_t *volinfo                   = NULL;
+
+        GF_ASSERT (THIS);
+        conf = THIS->private;
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        ret = glusterd_op_gsync_args_get (dict, op_errstr,
+                                          &volname, &slave, &host_uuid);
+        if (ret)
+                goto out;
+
+        snprintf (common_pem_file, sizeof(common_pem_file),
+                  "%s"GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Volinfo for %s"
+                        " (master) not found", volname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch slave volume name.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch slave IP.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
+                if (!ret && is_pem_push) {
+                        gf_log ("", GF_LOG_DEBUG, "Trying to setup"
+                                " pem files in slave");
+                        is_pem_push = 1;
+                } else
+                        is_pem_push = 0;
+
+                snprintf(hooks_args, sizeof(hooks_args),
+                         "is_push_pem=%d pub_file=%s slave_ip=%s",
+                         is_pem_push, common_pem_file, slave_ip);
+
+        } else
+                snprintf(hooks_args, sizeof(hooks_args),
+                         "This argument will stop the hooks script");
+
+        arg_buf = gf_strdup (hooks_args);
+        if (!arg_buf) {
+                gf_log ("", GF_LOG_ERROR, "Failed to"
+                        " gf_strdup");
+                if (is_force) {
+                        ret = 0;
+                        goto create_essentials;
+                }
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "hooks_args", arg_buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to set"
+                        " hooks_args in dict.");
+                if (is_force) {
+                        ret = 0;
+                        goto create_essentials;
+                }
+                goto out;
+        }
+
+create_essentials:
+
+        ret = glusterd_create_essential_dir_files (volinfo, dict, slave,
+                                                   slave_ip, slave_vol,
+                                                   op_errstr);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_slave_in_info (volinfo, slave,
+                                            host_uuid, op_errstr,
+                                            is_force);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to store"
+                          " slave info.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
         }
 
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index f446d2b58..12f3705b5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -43,6 +33,7 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
+#include "glusterd-locks.h"
 
 #include "glusterd1-xdr.h"
 #include "cli1-xdr.h"
@@ -57,6 +48,39 @@
 #include "defaults.c"
 #include "common-utils.h"
 
+#include "globals.h"
+#include "glusterd-syncop.h"
+
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
+extern glusterd_op_info_t opinfo;
+
+int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
+                                rpc_clnt_event_t event,
+                                void *data, rpc_clnt_notify_t notify_fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+        synclock_lock (&priv->big_lock);
+        ret = notify_fn (rpc, mydata, event, data);
+        synclock_unlock (&priv->big_lock);
+        return ret;
+}
+
+int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+
+        synclock_lock (&priv->big_lock);
+        ret = actor_fn (req);
+        synclock_unlock (&priv->big_lock);
+
+        return ret;
+}
+
 static int
 glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
                             char *hostname, int port,
@@ -78,10 +102,12 @@ glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
         ret = glusterd_friend_find (uuid, rhost, &peerinfo);
 
         if (ret) {
-                ret = glusterd_xfer_friend_add_resp (req, rhost, port, -1,
-                                                     GF_PROBE_UNKNOWN_PEER);
-                if (friend_req->vols.vols_val)
+                ret = glusterd_xfer_friend_add_resp (req, hostname, rhost, port,
+                                                     -1, GF_PROBE_UNKNOWN_PEER);
+                if (friend_req->vols.vols_val) {
                         free (friend_req->vols.vols_val);
+                        friend_req->vols.vols_val = NULL;
+                }
                 goto out;
         }
 
@@ -139,19 +165,16 @@ out:
         if (0 != ret) {
                 if (ctx && ctx->hostname)
                         GF_FREE (ctx->hostname);
-                if (ctx)
-                        GF_FREE (ctx);
+                GF_FREE (ctx);
                 if (dict) {
                         if ((!dict->extra_stdfree) &&
                             friend_req->vols.vols_val)
                                 free (friend_req->vols.vols_val);
                         dict_unref (dict);
                 } else {
-                    if (friend_req->vols.vols_val)
-                        free (friend_req->vols.vols_val);
+                    free (friend_req->vols.vols_val);
                 }
-                if (event)
-                        GF_FREE (event);
+                GF_FREE (event);
         } else {
                 if (peerinfo && (0 == peerinfo->connected))
                         ret = GLUSTERD_CONNECTION_AWAITED;
@@ -221,8 +244,7 @@ out:
         if (0 != ret) {
                 if (ctx && ctx->hostname)
                         GF_FREE (ctx->hostname);
-                if (ctx)
-                        GF_FREE (ctx);
+                GF_FREE (ctx);
         }
 
         return ret;
@@ -235,13 +257,14 @@ glusterd_add_peer_detail_to_dict (glusterd_peerinfo_t   *peerinfo,
 
         int             ret = -1;
         char            key[256] = {0, };
+        char           *peer_uuid_str = NULL;
 
         GF_ASSERT (peerinfo);
         GF_ASSERT (friends);
 
         snprintf (key, 256, "friend%d.uuid", count);
-        uuid_utoa_r (peerinfo->uuid, peerinfo->uuid_str);
-        ret = dict_set_str (friends, key, peerinfo->uuid_str);
+        peer_uuid_str = gd_peer_uuid_str (peerinfo);
+        ret = dict_set_str (friends, key, peer_uuid_str);
         if (ret)
                 goto out;
 
@@ -255,6 +278,11 @@ glusterd_add_peer_detail_to_dict (glusterd_peerinfo_t   *peerinfo,
         if (ret)
                 goto out;
 
+        snprintf (key, 256, "friend%d.stateId", count);
+        ret = dict_set_int32 (friends, key, peerinfo->state.state);
+        if (ret)
+                goto out;
+
         snprintf (key, 256, "friend%d.state", count);
         ret = dict_set_str (friends, key,
                     glusterd_friend_sm_state_name_get(peerinfo->state.state));
@@ -270,10 +298,47 @@ out:
         return ret;
 }
 
+struct args_pack {
+    dict_t *dict;
+    int vol_count;
+    int opt_count;
+};
+
+static int
+_build_option_key (dict_t *d, char *k, data_t *v, void *tmp)
+{
+        char                    reconfig_key[256] = {0, };
+        struct args_pack        *pack             = NULL;
+        int                     ret               = -1;
+        xlator_t                *this             = NULL;
+        glusterd_conf_t         *priv             = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        pack = tmp;
+        if (strcmp (k, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
+                return 0;
+
+        if (priv->op_version > GD_OP_VERSION_MIN) {
+                if ((strcmp (k, "features.limit-usage") == 0) ||
+                    (strcmp (k, "features.soft-limit") == 0))
+                        return 0;
+        }
+        snprintf (reconfig_key, 256, "volume%d.option.%s",
+                  pack->vol_count, k);
+        ret = dict_set_str (pack->dict, reconfig_key, v->data);
+        if (0 == ret)
+                pack->opt_count++;
+
+        return 0;
+}
 
 int
 glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
-                                    dict_t  *volumes, int   count)
+                                    dict_t  *volumes, int count)
 {
 
         int                     ret = -1;
@@ -281,19 +346,18 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
         glusterd_brickinfo_t    *brickinfo = NULL;
         char                    *buf = NULL;
         int                     i = 1;
-        data_pair_t             *pairs = NULL;
-        char                    reconfig_key[256] = {0, };
         dict_t                  *dict = NULL;
-        data_t                  *value = NULL;
-        int                     opt_count = 0;
         glusterd_conf_t         *priv = NULL;
         char                    *volume_id_str  = NULL;
-
+        struct args_pack        pack = {0,};
+        xlator_t                *this = NULL;
+        GF_UNUSED int           caps = 0;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (volumes);
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
 
         GF_ASSERT (priv);
 
@@ -347,12 +411,97 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
                 goto out;
 
         snprintf (key, 256, "volume%d.rebalance", count);
-        ret = dict_set_int32 (volumes, key, volinfo->defrag_cmd);
+        ret = dict_set_int32 (volumes, key, volinfo->rebal.defrag_cmd);
         if (ret)
                 goto out;
 
+#ifdef HAVE_BD_XLATOR
+        if (volinfo->caps) {
+                caps = 0;
+                snprintf (key, 256, "volume%d.xlator0", count);
+                buf = GF_MALLOC (256, gf_common_mt_char);
+                if (!buf) {
+                        ret = ENOMEM;
+                        goto out;
+                }
+                if (volinfo->caps & CAPS_BD)
+                        snprintf (buf, 256, "BD");
+                ret = dict_set_dynstr (volumes, key, buf);
+                if (ret) {
+                        GF_FREE (buf);
+                        goto out;
+                }
+
+                if (volinfo->caps & CAPS_THIN) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "thin");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret) {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_COPY) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_copy");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret) {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_SNAPSHOT) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_snapshot");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)  {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_ZERO) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_zerofill");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)  {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+        }
+#endif
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 char    brick[1024] = {0,};
+                char    brick_uuid[64] = {0,};
                 snprintf (key, 256, "volume%d.brick%d", count, i);
                 snprintf (brick, 1024, "%s:%s", brickinfo->hostname,
                           brickinfo->path);
@@ -360,6 +509,25 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
                 ret = dict_set_dynstr (volumes, key, buf);
                 if (ret)
                         goto out;
+                snprintf (key, 256, "volume%d.brick%d.uuid", count, i);
+                snprintf (brick_uuid, 64, "%s", uuid_utoa (brickinfo->uuid));
+                buf = gf_strdup (brick_uuid);
+                if (!buf)
+                        goto out;
+                ret = dict_set_dynstr (volumes, key, buf);
+                if (ret)
+                        goto out;
+
+#ifdef HAVE_BD_XLATOR
+                if (volinfo->caps & CAPS_BD) {
+                        snprintf (key, 256, "volume%d.vg%d", count, i);
+                        snprintf (brick, 1024, "%s", brickinfo->vg);
+                        buf = gf_strdup (brick);
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)
+                                goto out;
+                }
+#endif
                 i++;
         }
 
@@ -369,25 +537,14 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
                 goto out;
         }
 
-        pairs = dict->members_list;
-
-        while (pairs) {
-                if (1 == glusterd_check_option_exists (pairs->key, NULL)) {
-                        value = pairs->value;
-                        if (!value)
-                                continue;
-
-                        snprintf (reconfig_key, 256, "volume%d.option.%s", count,
-                                  pairs->key);
-                        ret = dict_set_str  (volumes, reconfig_key, value->data);
-                        if (!ret)
-                            opt_count++;
-                }
-                pairs = pairs->next;
-        }
+        pack.dict = volumes;
+        pack.vol_count = count;
+        pack.opt_count = 0;
+        dict_foreach (dict, _build_option_key, (void *) &pack);
+        dict_foreach (priv->opts, _build_option_key, &pack);
 
-        snprintf (key, 256, "volume%d.opt_count", count);
-        ret = dict_set_int32 (volumes, key, opt_count);
+        snprintf (key, 256, "volume%d.opt_count", pack.vol_count);
+        ret = dict_set_int32 (volumes, key, pack.opt_count);
 out:
         return ret;
 }
@@ -397,13 +554,18 @@ glusterd_friend_find (uuid_t uuid, char *hostname,
                       glusterd_peerinfo_t **peerinfo)
 {
         int     ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         if (uuid) {
                 ret = glusterd_friend_find_by_uuid (uuid, peerinfo);
 
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_INFO,
-                                 "Unable to find peer by uuid");
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                 "Unable to find peer by uuid: %s",
+                                 uuid_utoa (uuid));
                 } else {
                         goto out;
                 }
@@ -414,7 +576,7 @@ glusterd_friend_find (uuid_t uuid, char *hostname,
                 ret = glusterd_friend_find_by_hostname (hostname, peerinfo);
 
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_INFO,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "Unable to find hostname: %s", hostname);
                 } else {
                         goto out;
@@ -426,73 +588,185 @@ out:
 }
 
 int32_t
-glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx)
+glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                       char *err_str, size_t err_len)
 {
-        int32_t                  ret    = -1;
-        xlator_t                *this   = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int32_t                  locked = 0;
+        int32_t                     ret             = -1;
+        dict_t                     *dict            = NULL;
+        xlator_t                   *this            = NULL;
+        glusterd_conf_t            *priv            = NULL;
+        int32_t                     locked          = 0;
+        char                       *tmp             = NULL;
+        char                       *volname         = NULL;
+        uuid_t                     *txn_id          = NULL;
+        glusterd_op_info_t          txn_op_info     = {{0},};
+        glusterd_op_sm_event_type_t event_type      = GD_OP_EVENT_NONE;
 
         GF_ASSERT (req);
         GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
         GF_ASSERT (NULL != ctx);
 
         this = THIS;
+        GF_ASSERT (this);
         priv = this->private;
         GF_ASSERT (priv);
 
-        ret = glusterd_lock (priv->uuid);
+        dict = ctx;
+
+        /* Generate a transaction-id for this operation and
+         * save it in the dict. This transaction id distinguishes
+         * each transaction, and helps separate opinfos in the
+         * op state machine. */
+        ret = glusterd_generate_txn_id (dict, &txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to generate transaction id");
+                goto out;
+        }
 
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        ret = glusterd_set_originator_uuid (dict);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to acquire local lock, ret: %d", ret);
+                        "Failed to set originator_uuid.");
                 goto out;
         }
 
+        /* Based on the op_version, acquire a cluster or volume lock */
+        if (priv->op_version < GD_OP_VERSION_4) {
+                ret = glusterd_lock (MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock on localhost, ret: %d",
+                                ret);
+                        snprintf (err_str, err_len,
+                                  "Another transaction is in progress. "
+                                  "Please try again after sometime.");
+                        goto out;
+                }
+        } else {
+                /* If no volname is given as a part of the command, locks will
+                 * not be held */
+                ret = dict_get_str (dict, "volname", &tmp);
+                if (ret) {
+                        gf_log ("", GF_LOG_INFO,
+                                "No Volume name present. "
+                                "Locks not being held.");
+                        goto local_locking_done;
+                } else {
+                        /* Use a copy of volname, as cli response will be
+                         * sent before the unlock, and the volname in the
+                         * dict, might be removed */
+                        volname = gf_strdup (tmp);
+                        if (!volname)
+                                goto out;
+                }
+
+                ret = glusterd_volume_lock (volname, MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock for %s", volname);
+                        snprintf (err_str, err_len,
+                                  "Another transaction is in progress for %s. "
+                                  "Please try again after sometime.", volname);
+                        goto out;
+                }
+        }
+
         locked = 1;
-        gf_log (this->name, GF_LOG_INFO, "Acquired local lock");
+        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock on localhost");
+
+local_locking_done:
+
+        /* If no volname is given as a part of the command, locks will
+         * not be held, hence sending stage event. */
+        if (volname || (priv->op_version < GD_OP_VERSION_4))
+                event_type = GD_OP_EVENT_START_LOCK;
+        else {
+                txn_op_info.state.state = GD_OP_STATE_LOCK_SENT;
+                event_type = GD_OP_EVENT_ALL_ACC;
+        }
+
+        /* Save opinfo for this transaction with the transaction id */
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, ctx, req);
+
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                if (ctx)
+                        dict_unref (ctx);
+                goto out;
+        }
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_START_LOCK, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, ctx);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to acquire cluster"
                         " lock.");
                 goto out;
         }
 
-        glusterd_op_set_op (op);
-        glusterd_op_set_ctx (ctx);
-        glusterd_op_set_req (req);
-
-
 out:
-        if (locked && ret)
-                glusterd_unlock (priv->uuid);
+        if (locked && ret) {
+                /* Based on the op-version, we release the
+                 * cluster or volume lock */
+                if (priv->op_version < GD_OP_VERSION_4)
+                        glusterd_unlock (MY_UUID);
+                else {
+                        ret = glusterd_volume_unlock (volname, MY_UUID);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                        ret = -1;
+                }
+        }
+
+        if (volname)
+                GF_FREE (volname);
 
         gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+__glusterd_handle_cluster_lock (rpcsvc_request_t *req)
 {
-        gd1_mgmt_cluster_lock_req       lock_req = {{0},};
-        int32_t                         ret = -1;
-        glusterd_op_lock_ctx_t          *ctx = NULL;
-        glusterd_peerinfo_t             *peerinfo = NULL;
+        dict_t                         *op_ctx      = NULL;
+        int32_t                         ret         = -1;
+        gd1_mgmt_cluster_lock_req       lock_req    = {{0},};
+        glusterd_op_lock_ctx_t         *ctx         = NULL;
+        glusterd_op_t                   op          = GD_OP_EVENT_LOCK;
+        glusterd_peerinfo_t            *peerinfo    = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+        glusterd_conf_t                *priv        = NULL;
+        uuid_t                         *txn_id      = NULL;
+        xlator_t                       *this        = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &lock_req, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req)) {
-                //failed to decode msg;
+        txn_id = &priv->global_txn_id;
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "request received from peer");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received LOCK from uuid: %s", uuid_utoa (lock_req.uuid));
+        gf_log (this->name, GF_LOG_DEBUG, "Received LOCK from uuid: %s",
+                uuid_utoa (lock_req.uuid));
 
         if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
-                gf_log (THIS->name, GF_LOG_WARNING, "%s doesn't "
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
                         "belong to the cluster. Ignoring request.",
                         uuid_utoa (lock_req.uuid));
                 ret = -1;
@@ -508,19 +782,229 @@ glusterd_handle_cluster_lock (rpcsvc_request_t *req)
 
         uuid_copy (ctx->uuid, lock_req.uuid);
         ctx->req = req;
+        ctx->dict = NULL;
+
+        op_ctx =  dict_new ();
+        if (!op_ctx) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set new dict");
+                goto out;
+        }
+
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, op_ctx, req);
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, ctx);
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                dict_unref (txn_op_info.op_ctx);
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_LOCK");
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cluster_lock);
+}
+
+static int
+glusterd_handle_volume_lock_fn (rpcsvc_request_t *req)
+{
+        gd1_mgmt_volume_lock_req        lock_req    = {{0},};
+        int32_t                         ret         = -1;
+        glusterd_op_lock_ctx_t         *ctx         = NULL;
+        glusterd_peerinfo_t            *peerinfo    = NULL;
+        xlator_t                       *this        = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_volume_lock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received volume lock req "
+                "from uuid: %s txn_id: %s", uuid_utoa (lock_req.uuid),
+                uuid_utoa (lock_req.txn_id));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+
+        ctx->dict = dict_new ();
+        if (!ctx->dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (lock_req.dict.dict_val,
+                                lock_req.dict.dict_len, &ctx->dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &lock_req.op,
+                                  ctx->dict, req);
+
+        ret = glusterd_set_txn_opinfo (&lock_req.txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK,
+                                           &lock_req.txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_LOCK");
+
+out:
+        if (ret) {
+                if (ctx) {
+                        if (ctx->dict)
+                                dict_destroy (ctx->dict);
+                        GF_FREE (ctx);
+                }
+        }
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
 
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
+glusterd_handle_volume_lock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_volume_lock_fn);
+}
+
+static int
+glusterd_handle_volume_unlock_fn (rpcsvc_request_t *req)
+{
+        gd1_mgmt_volume_unlock_req        lock_req = {{0},};
+        int32_t                           ret      = -1;
+        glusterd_op_lock_ctx_t           *ctx      = NULL;
+        glusterd_peerinfo_t              *peerinfo = NULL;
+        xlator_t                         *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_volume_unlock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received volume unlock req "
+                "from uuid: %s", uuid_utoa (lock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+
+        ctx->dict = dict_new ();
+        if (!ctx->dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (lock_req.dict.dict_val,
+                                lock_req.dict.dict_len, &ctx->dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK,
+                                           &lock_req.txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_UNLOCK");
+
+out:
+        if (ret) {
+                if (ctx) {
+                        if (ctx->dict)
+                                dict_destroy (ctx->dict);
+                        GF_FREE (ctx);
+                }
+        }
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_handle_volume_unlock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_volume_unlock_fn);
+}
+
+int
 glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
                          glusterd_op_t op, uuid_t uuid,
                          char *buf_val, size_t buf_len,
@@ -531,10 +1015,13 @@ glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
         char                str[50] = {0,};
         glusterd_req_ctx_t *req_ctx = NULL;
         dict_t             *dict    = NULL;
+        xlator_t           *this    = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         uuid_unparse (uuid, str);
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received op from uuid: %s", str);
+        gf_log (this->name, GF_LOG_DEBUG, "Received op from uuid %s", str);
 
         dict = dict_new ();
         if (!dict)
@@ -549,7 +1036,7 @@ glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
         req_ctx->op = op;
         ret = dict_unserialize (buf_val, buf_len, &dict);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_WARNING,
                         "failed to unserialize the dictionary");
                 goto out;
         }
@@ -562,29 +1049,43 @@ out:
         if (ret) {
                 if (dict)
                         dict_unref (dict);
-                if (req_ctx)
-                        GF_FREE (req_ctx);
+                GF_FREE (req_ctx);
         }
         return ret;
 }
 
 int
-glusterd_handle_stage_op (rpcsvc_request_t *req)
+__glusterd_handle_stage_op (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         glusterd_req_ctx_t              *req_ctx = NULL;
         gd1_mgmt_stage_op_req           op_req = {{0},};
         glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+        glusterd_op_sm_state_info_t     state = {0,};
+        glusterd_conf_t                 *priv = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
         GF_ASSERT (req);
-        if (!xdr_to_generic (req->msg[0], &op_req, (xdrproc_t)xdr_gd1_mgmt_stage_op_req)) {
-                //failed to decode msg;
+
+        txn_id = &priv->global_txn_id;
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode stage "
+                        "request received from peer");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
         if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
-                gf_log (THIS->name, GF_LOG_WARNING, "%s doesn't "
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
                         "belong to the cluster. Ignoring request.",
                         uuid_utoa (op_req.uuid));
                 ret = -1;
@@ -597,34 +1098,81 @@ glusterd_handle_stage_op (rpcsvc_request_t *req)
         if (ret)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP, req_ctx);
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        /* In cases where there is no volname, the receivers won't have a
+         * transaction opinfo created, as for those operations, the locking
+         * phase where the transaction opinfos are created, won't be called. */
+        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No transaction's opinfo set");
+
+                state.state = GD_OP_STATE_LOCKED;
+                glusterd_txn_opinfo_init (&txn_op_info, &state,
+                                          &op_req.op, req_ctx->dict, req);
+
+                ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set transaction's opinfo");
+                        dict_unref (req_ctx->dict);
+                        goto out;
+                }
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP,
+                                           txn_id, req_ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_STAGE_OP");
 
  out:
-        if (op_req.buf.buf_val)
-                free (op_req.buf.buf_val);//malloced by xdr
+        free (op_req.buf.buf_val);//malloced by xdr
         glusterd_friend_sm ();
         glusterd_op_sm ();
         return ret;
 }
 
 int
-glusterd_handle_commit_op (rpcsvc_request_t *req)
+glusterd_handle_stage_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_stage_op);
+}
+
+
+int
+__glusterd_handle_commit_op (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         glusterd_req_ctx_t              *req_ctx = NULL;
         gd1_mgmt_commit_op_req          op_req = {{0},};
         glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = NULL;
+        glusterd_conf_t                 *priv = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &op_req, (xdrproc_t)xdr_gd1_mgmt_commit_op_req)) {
-                //failed to decode msg;
+        txn_id = &priv->global_txn_id;
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "request received from peer");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
         if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
-                gf_log (THIS->name, GF_LOG_WARNING, "%s doesn't "
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
                         "belong to the cluster. Ignoring request.",
                         uuid_utoa (op_req.uuid));
                 ret = -1;
@@ -639,70 +1187,124 @@ glusterd_handle_commit_op (rpcsvc_request_t *req)
         if (ret)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP, req_ctx);
-        if (ret)
-                goto out;
-        ret = glusterd_op_init_ctx (op_req.op);
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP,
+                                           txn_id, req_ctx);
 
 out:
-        if (op_req.buf.buf_val)
-                free (op_req.buf.buf_val);//malloced by xdr
+        free (op_req.buf.buf_val);//malloced by xdr
         glusterd_friend_sm ();
         glusterd_op_sm ();
         return ret;
 }
 
 int
-glusterd_handle_cli_probe (rpcsvc_request_t *req)
+glusterd_handle_commit_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_commit_op);
+}
+
+int
+__glusterd_handle_cli_probe (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
-        gf1_cli_probe_req               cli_req = {0,};
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        gf_boolean_t                    run_fsm = _gf_true;
+        gf_cli_req                  cli_req = {{0,},};
+        glusterd_peerinfo_t       *peerinfo = NULL;
+        gf_boolean_t                run_fsm = _gf_true;
+        xlator_t                      *this = NULL;
+        char                     *bind_name = NULL;
+        dict_t                        *dict = NULL;
+        char                      *hostname = NULL;
+        int                            port = 0;
+
         GF_ASSERT (req);
+        this = THIS;
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf1_cli_probe_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0)  {
                 //failed to decode msg;
                 gf_log ("", GF_LOG_ERROR, "xdr decoding error");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
-        gf_cmd_log ("peer probe", " on host %s:%d", cli_req.hostname,
-                    cli_req.port);
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len, &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hostname");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "port", &port);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get port");
+                goto out;
+        }
+
+        if (glusterd_is_any_volume_in_server_quorum (this) &&
+            !does_gd_meet_server_quorum (this)) {
+                glusterd_xfer_cli_probe_resp (req, -1, GF_PROBE_QUORUM_NOT_MET,
+                                              NULL, hostname, port, dict);
+                gf_log (this->name, GF_LOG_ERROR, "Quorum does not meet, "
+                        "rejecting operation");
+                ret = 0;
+                goto out;
+        }
+
         gf_log ("glusterd", GF_LOG_INFO, "Received CLI probe req %s %d",
-                cli_req.hostname, cli_req.port);
+                hostname, port);
 
-        if (!(ret = glusterd_is_local_addr(cli_req.hostname))) {
+        if (dict_get_str(this->options,"transport.socket.bind-address",
+                         &bind_name) == 0) {
+                gf_log ("glusterd", GF_LOG_DEBUG,
+                        "only checking probe address vs. bind address");
+                ret = gf_is_same_address (bind_name, hostname);
+        }
+        else {
+                ret = gf_is_local_addr (hostname);
+        }
+        if (ret) {
                 glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST,
-                                              cli_req.hostname, cli_req.port);
+                                              NULL, hostname, port, dict);
+                ret = 0;
                 goto out;
         }
 
-        if (!(ret = glusterd_friend_find_by_hostname(cli_req.hostname,
-                                         &peerinfo))) {
-                if (strcmp (peerinfo->hostname, cli_req.hostname) == 0) {
+        if (!(ret = glusterd_friend_find_by_hostname (hostname, &peerinfo))) {
+                if (strcmp (peerinfo->hostname, hostname) == 0) {
 
-                        gf_log ("glusterd", GF_LOG_DEBUG, "Probe host %s port %d"
-                               " already a peer", cli_req.hostname, cli_req.port);
+                        gf_log ("glusterd", GF_LOG_DEBUG, "Probe host %s port "
+                                "%d already a peer", hostname, port);
                         glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND,
-                                                      cli_req.hostname, cli_req.port);
+                                                      NULL, hostname, port,
+                                                      dict);
                         goto out;
                 }
         }
-        ret = glusterd_probe_begin (req, cli_req.hostname, cli_req.port);
-
-        gf_cmd_log ("peer probe","on host %s:%d %s",cli_req.hostname, cli_req.port,
-                    (ret) ? "FAILED" : "SUCCESS");
+        ret = glusterd_probe_begin (req, hostname, port, dict);
 
         if (ret == GLUSTERD_CONNECTION_AWAITED) {
                 //fsm should be run after connection establishes
                 run_fsm = _gf_false;
                 ret = 0;
         }
+
 out:
-        if (cli_req.hostname)
-                free (cli_req.hostname);//its malloced by xdr
+        free (cli_req.dict.dict_val);
 
         if (run_fsm) {
                 glusterd_friend_sm ();
@@ -713,14 +1315,24 @@ out:
 }
 
 int
-glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+glusterd_handle_cli_probe (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_cli_probe);
+}
+
+int
+__glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
-        gf1_cli_deprobe_req               cli_req = {0,};
-        uuid_t                          uuid = {0};
-        int                             op_errno = 0;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *priv = NULL;
+        gf_cli_req                  cli_req = {{0,},};
+        uuid_t                         uuid = {0};
+        int                        op_errno = 0;
+        xlator_t                      *this = NULL;
+        glusterd_conf_t               *priv = NULL;
+        dict_t                        *dict = NULL;
+        char                      *hostname = NULL;
+        int                            port = 0;
+        int                           flags = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -728,62 +1340,99 @@ glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
         GF_ASSERT (priv);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-            (xdrproc_t)xdr_gf1_cli_deprobe_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len, &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                }
+        }
+
         gf_log ("glusterd", GF_LOG_INFO, "Received CLI deprobe req");
 
-        ret = glusterd_hostname_to_uuid (cli_req.hostname, uuid);
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hostname");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "port", &port);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get port");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "flags", &flags);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get flags");
+                goto out;
+        }
+
+        ret = glusterd_hostname_to_uuid (hostname, uuid);
         if (ret) {
                 op_errno = GF_DEPROBE_NOT_FRIEND;
                 goto out;
         }
 
-        if (!uuid_compare (uuid, priv->uuid)) {
+        if (!uuid_compare (uuid, MY_UUID)) {
                 op_errno = GF_DEPROBE_LOCALHOST;
                 ret = -1;
                 goto out;
         }
 
-        if (!uuid_is_null (uuid) && !(cli_req.flags & GF_CLI_FLAG_OP_FORCE)) {
-                /* Check if peers are connected, except peer being detached*/
-                if (!glusterd_chk_peers_connected_befriended (uuid)) {
-                        ret = -1;
-                        op_errno = GF_DEPROBE_FRIEND_DOWN;
-                        goto out;
+        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+                if (!uuid_is_null (uuid)) {
+                        /* Check if peers are connected, except peer being detached*/
+                        if (!glusterd_chk_peers_connected_befriended (uuid)) {
+                                ret = -1;
+                                op_errno = GF_DEPROBE_FRIEND_DOWN;
+                                goto out;
+                        }
+                        ret = glusterd_all_volume_cond_check (
+                                                 glusterd_friend_brick_belongs,
+                                                 -1, &uuid);
+                        if (ret) {
+                                op_errno = GF_DEPROBE_BRICK_EXIST;
+                                goto out;
+                        }
                 }
-                ret = glusterd_all_volume_cond_check (
-                                                glusterd_friend_brick_belongs,
-                                                -1, &uuid);
-                if (ret) {
-                        op_errno = GF_DEPROBE_BRICK_EXIST;
+
+                if (glusterd_is_any_volume_in_server_quorum (this) &&
+                    !does_gd_meet_server_quorum (this)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Quorum does not "
+                                "meet, rejecting operation");
+                        ret = -1;
+                        op_errno = GF_DEPROBE_QUORUM_NOT_MET;
                         goto out;
                 }
         }
 
         if (!uuid_is_null (uuid)) {
-                ret = glusterd_deprobe_begin (req, cli_req.hostname,
-                                              cli_req.port, uuid);
+                ret = glusterd_deprobe_begin (req, hostname, port, uuid, dict);
         } else {
-                ret = glusterd_deprobe_begin (req, cli_req.hostname,
-                                              cli_req.port, NULL);
+                ret = glusterd_deprobe_begin (req, hostname, port, NULL, dict);
         }
 
-        gf_cmd_log ("peer deprobe", "on host %s:%d %s", cli_req.hostname,
-                    cli_req.port, (ret) ? "FAILED" : "SUCCESS");
 out:
+        free (cli_req.dict.dict_val);
+
         if (ret) {
-                ret = glusterd_xfer_cli_deprobe_resp (req, ret, op_errno,
-                                                      cli_req.hostname);
+                ret = glusterd_xfer_cli_deprobe_resp (req, ret, op_errno, NULL,
+                                                      hostname, dict);
         }
 
-        if (cli_req.hostname)
-                free (cli_req.hostname);//malloced by xdr
-
         glusterd_friend_sm ();
         glusterd_op_sm ();
 
@@ -791,7 +1440,13 @@ out:
 }
 
 int
-glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_cli_deprobe);
+}
+
+int
+__glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf1_cli_peer_list_req           cli_req = {0,};
@@ -799,7 +1454,9 @@ glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf1_cli_peer_list_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf1_cli_peer_list_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -837,7 +1494,14 @@ out:
 }
 
 int
-glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_list_friends);
+}
+
+int
+__glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -846,7 +1510,8 @@ glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -890,7 +1555,216 @@ out:
 }
 
 int
-glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_get_volume);
+}
+
+int
+__glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+{
+        int                     ret     = -1;
+        dict_t                  *dict   = NULL;
+        xlator_t                *this   = NULL;
+        glusterd_conf_t         *priv   = NULL;
+        uuid_t                  uuid    = {0};
+        gf_cli_rsp              rsp     = {0,};
+        gf_cli_req              cli_req = {{0,}};
+        char                    msg_str[2048] = {0,};
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Received uuid reset req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
+                                  "the buffer");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        /* In the above section if dict_unserialize is successful, ret is set
+         * to zero.
+         */
+        ret = -1;
+        // Do not allow peer reset if there are any volumes in the cluster
+        if (!list_empty (&priv->volumes)) {
+                snprintf (msg_str, sizeof (msg_str), "volumes are already "
+                          "present in the cluster. Resetting uuid is not "
+                          "allowed");
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg_str);
+                goto out;
+        }
+
+        // Do not allow peer reset if trusted storage pool is already formed
+        if (!list_empty (&priv->peers)) {
+                snprintf (msg_str, sizeof (msg_str),"trusted storage pool "
+                          "has been already formed. Please detach this peer "
+                          "from the pool and reset its uuid.");
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg_str);
+                goto out;
+        }
+
+        uuid_copy (uuid, priv->uuid);
+        ret = glusterd_uuid_generate_save ();
+
+        if (!uuid_compare (uuid, MY_UUID)) {
+                snprintf (msg_str, sizeof (msg_str), "old uuid and the new uuid"
+                          " are same. Try gluster peer reset again");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg_str);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                if (msg_str[0] == '\0')
+                        snprintf (msg_str, sizeof (msg_str), "Operation "
+                                  "failed");
+                rsp.op_errstr = msg_str;
+                ret = 0;
+        } else {
+                rsp.op_errstr = "";
+        }
+
+        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
+                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_uuid_reset);
+}
+
+int
+__glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+{
+        int                     ret         = -1;
+        dict_t                  *dict       = NULL;
+        dict_t                  *rsp_dict   = NULL;
+        xlator_t                *this       = NULL;
+        glusterd_conf_t         *priv       = NULL;
+        gf_cli_rsp              rsp         = {0,};
+        gf_cli_req              cli_req     = {{0,}};
+        char                    msg_str[2048] = {0,};
+        char                    uuid_str[64] = {0,};
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Received uuid get req");
+
+        if (cli_req.dict.dict_len) {
+                dict  = dict_new ();
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
+                                  "the buffer");
+                        goto out;
+
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+
+                }
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        ret = dict_set_str (rsp_dict, "uuid", uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set uuid in "
+                        "dictionary.");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize "
+                        "dictionary.");
+                goto out;
+        }
+        ret = 0;
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                if (msg_str[0] == '\0')
+                        snprintf (msg_str, sizeof (msg_str), "Operation "
+                                  "failed");
+                rsp.op_errstr = msg_str;
+
+        } else {
+                rsp.op_errstr = "";
+
+        }
+
+        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
+                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+
+        return 0;
+}
+int
+glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_uuid_get);
+}
+
+int
+__glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
 {
         int                     ret = -1;
         dict_t                  *dict = NULL;
@@ -923,7 +1797,7 @@ glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
                 goto out;
 
         ret = dict_allocate_and_serialize (dict, &rsp.dict.dict_val,
-                                           (size_t *)&rsp.dict.dict_len);
+                                           &rsp.dict.dict_len);
         if (ret)
                 goto out;
 
@@ -936,42 +1810,57 @@ out:
         else
                 rsp.op_errstr = "";
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gf_cli_rsp);
+        ret = 0;
 
         if (dict)
                 dict_unref (dict);
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
+
         return ret;
 }
 
+int
+glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_list_volume);
+}
+
 int32_t
-glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx)
+glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                   char *err_str, size_t err_len)
 {
         int             ret = -1;
 
-        ret = glusterd_op_txn_begin (req, op, ctx);
+        ret = glusterd_op_txn_begin (req, op, ctx, err_str, err_len);
 
         return ret;
 }
 
-
-
 int
-glusterd_handle_reset_volume (rpcsvc_request_t *req)
+__glusterd_handle_reset_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
         dict_t                          *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_RESET_VOLUME;
         char                            *volname = NULL;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -984,8 +1873,10 @@ glusterd_handle_reset_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                     "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -994,31 +1885,37 @@ glusterd_handle_reset_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
+        gf_log (this->name, GF_LOG_DEBUG, "Received volume reset request for "
+                "volume %s", volname);
 
-        gf_cmd_log ("Volume reset", "volume  : %s", volname);
-        ret = glusterd_op_begin (req, GD_OP_RESET_VOLUME, dict);
-        gf_cmd_log ("Volume reset", " on volume %s %s ", volname,
-                ((ret == 0)? " SUCCEDED":" FAILED"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_RESET_VOLUME, dict);
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
 
         return ret;
 }
 
+int
+glusterd_handle_reset_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_reset_volume);
+}
 
 int
-glusterd_handle_set_volume (rpcsvc_request_t *req)
+__glusterd_handle_set_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -1027,11 +1924,21 @@ glusterd_handle_set_volume (rpcsvc_request_t *req)
         char                            *key = NULL;
         char                            *value = NULL;
         char                            *volname = NULL;
+        char                            *op_errstr = NULL;
+        gf_boolean_t                    help = _gf_false;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode "
+                          "request received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -1044,9 +1951,11 @@ glusterd_handle_set_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -1055,65 +1964,81 @@ glusterd_handle_set_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to get volume name, while"
-                        "handling volume set command");
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name while handling volume set command");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if (strcmp (volname, "help") == 0 ||
+            strcmp (volname, "help-xml") == 0) {
+                ret = glusterd_volset_help (dict, &op_errstr);
+                help = _gf_true;
                 goto out;
         }
 
         ret = dict_get_str (dict, "key1", &key);
         if (ret) {
-                if (strcmp (volname, "help-xml") && strcmp (volname, "help")) {
-                        gf_log ("", GF_LOG_WARNING, "Unable to get key, while "
-                                "handling volume set for %s",volname);
-                        goto out;
-                }
+                snprintf (err_str, sizeof (err_str), "Failed to get key while"
+                          " handling volume set for %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
         }
 
         ret = dict_get_str (dict, "value1", &value);
         if (ret) {
-                if (strcmp (volname, "help-xml") && strcmp (volname, "help")) {
-                        gf_log ("", GF_LOG_WARNING, "Unable to get value, while"
-                                "handling volume set for %s",volname);
-                        goto out;
-                }
+                snprintf (err_str, sizeof (err_str), "Failed to get value while"
+                          " handling volume set for %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
         }
+        gf_log (this->name, GF_LOG_DEBUG, "Received volume set request for "
+                "volume %s", volname);
 
+        ret = glusterd_op_begin_synctask (req, GD_OP_SET_VOLUME, dict);
 
-        gf_cmd_log ("volume set", "volume-name:%s: key:%s, value:%s",volname,
-                    key, value);
-        ret = glusterd_op_begin (req, GD_OP_SET_VOLUME, dict);
-        gf_cmd_log ("volume set", "volume-name:%s: key:%s, value:%s %s",
-                    volname, key, value, (ret == 0)? "SUCCEDED" : "FAILED" );
 out:
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        if (ret) {
-                if (dict)
-                        dict_unref (dict);
+        if (help)
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req, dict,
+                                                     (op_errstr)? op_errstr:"");
+        else if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
+        if (op_errstr)
+                GF_FREE (op_errstr);
+
         return ret;
 }
 
 int
-glusterd_handle_sync_volume (rpcsvc_request_t *req)
+glusterd_handle_set_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_set_volume);
+}
+
+int
+__glusterd_handle_sync_volume (rpcsvc_request_t *req)
 {
         int32_t                          ret     = -1;
         gf_cli_req                       cli_req = {{0,}};
         dict_t                           *dict = NULL;
         gf_cli_rsp                       cli_rsp = {0.};
         char                             msg[2048] = {0,};
-        glusterd_volinfo_t               *volinfo = NULL;
         char                             *volname = NULL;
         gf1_cli_sync_volume              flags = 0;
         char                             *hostname = NULL;
+        xlator_t                         *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1127,9 +2052,11 @@ glusterd_handle_sync_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -1138,7 +2065,8 @@ glusterd_handle_sync_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "hostname", &hostname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get hostname");
+                snprintf (msg, sizeof (msg), "Failed to get hostname");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
@@ -1146,48 +2074,25 @@ glusterd_handle_sync_volume (rpcsvc_request_t *req)
         if (ret) {
                 ret = dict_get_int32 (dict, "flags", (int32_t*)&flags);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Unable to get volume"
-                                "name, or flags");
+                        snprintf (msg, sizeof (msg), "Failed to get volume name"
+                                  " or flags");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                         goto out;
                 }
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received volume sync req "
-                "for volume %s",
-                (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
+        gf_log (this->name, GF_LOG_INFO, "Received volume sync req "
+                "for volume %s", (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
 
-        if (!glusterd_is_local_addr (hostname)) {
+        if (gf_is_local_addr (hostname)) {
                 ret = -1;
                 snprintf (msg, sizeof (msg), "sync from localhost"
                           " not allowed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
-        if (!flags) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (!ret) {
-                        snprintf (msg, sizeof (msg), "please delete the "
-                                 "volume: %s before sync", volname);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynmstr (dict, "volname", volname);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "volume name set failed");
-                        snprintf (msg, sizeof (msg), "volume name set failed");
-                        goto out;
-                }
-        } else {
-                if (glusterd_volume_count_get ()) {
-                        snprintf (msg, sizeof (msg), "please delete all the "
-                                 "volumes before full sync");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        ret = glusterd_op_begin (req, GD_OP_SYNC_VOLUME, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_SYNC_VOLUME, dict);
 
 out:
         if (ret) {
@@ -1195,22 +2100,22 @@ out:
                 cli_rsp.op_errstr = msg;
                 if (msg[0] == '\0')
                         snprintf (msg, sizeof (msg), "Operation failed");
-                glusterd_submit_reply(req, &cli_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gf_cli_rsp);
-                if (dict)
-                        dict_unref (dict);
-
+                glusterd_to_cli (req, &cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
 
                 ret = 0; //sent error to cli, prevent second reply
         }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         return ret;
 }
 
 int
+glusterd_handle_sync_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_sync_volume);
+}
+
+int
 glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
                             char *op_errstr, dict_t *dict)
 {
@@ -1225,12 +2130,11 @@ glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
         rsp.op_errstr = op_errstr;
         if (rsp.op_ret == 0)
                 ret = dict_allocate_and_serialize (dict, &rsp.fsm_log.fsm_log_val,
-                                                (size_t *)&rsp.fsm_log.fsm_log_len);
+                                                &rsp.fsm_log.fsm_log_len);
 
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
-        if (rsp.fsm_log.fsm_log_val)
-                GF_FREE (rsp.fsm_log.fsm_log_val);
+        GF_FREE (rsp.fsm_log.fsm_log_val);
 
         gf_log ("glusterd", GF_LOG_DEBUG, "Responded, ret: %d", ret);
 
@@ -1238,7 +2142,7 @@ glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
 }
 
 int
-glusterd_handle_fsm_log (rpcsvc_request_t *req)
+__glusterd_handle_fsm_log (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf1_cli_fsm_log_req             cli_req = {0,};
@@ -1251,7 +2155,9 @@ glusterd_handle_fsm_log (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf1_cli_fsm_log_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf1_cli_fsm_log_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 snprintf (msg, sizeof (msg), "Garbage request");
@@ -1282,8 +2188,7 @@ glusterd_handle_fsm_log (rpcsvc_request_t *req)
         ret = glusterd_sm_tr_log_add_to_dict (dict, log);
 out:
         (void)glusterd_fsm_log_send_resp (req, ret, msg, dict);
-        if (cli_req.name)
-                free (cli_req.name);//malloced by xdr
+        free (cli_req.name);//malloced by xdr
         if (dict)
                 dict_unref (dict);
 
@@ -1294,6 +2199,12 @@ out:
 }
 
 int
+glusterd_handle_fsm_log (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_fsm_log);
+}
+
+int
 glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
 {
 
@@ -1307,8 +2218,7 @@ glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded, ret: %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to lock, ret: %d", ret);
 
         return 0;
 }
@@ -1327,35 +2237,95 @@ glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to unlock, ret: %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to unlock, ret: %d", ret);
 
         return ret;
 }
 
 int
-glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+glusterd_op_volume_lock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+                                   int32_t status)
+{
+
+        gd1_mgmt_volume_lock_rsp       rsp = {{0},};
+        int                            ret = -1;
+
+        GF_ASSERT (req);
+        GF_ASSERT (txn_id);
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+        uuid_copy (rsp.txn_id, *txn_id);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_volume_lock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to volume lock, ret: %d",
+                ret);
+
+        return ret;
+}
+
+int
+glusterd_op_volume_unlock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+                                     int32_t status)
+{
+
+        gd1_mgmt_volume_unlock_rsp      rsp = {{0},};
+        int                             ret = -1;
+
+        GF_ASSERT (req);
+        GF_ASSERT (txn_id);
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+        glusterd_get_uuid (&rsp.uuid);
+        uuid_copy (rsp.txn_id, *txn_id);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_volume_unlock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to volume unlock, ret: %d",
+                ret);
+
+        return ret;
+}
+
+int
+__glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
 {
         gd1_mgmt_cluster_unlock_req     unlock_req = {{0}, };
         int32_t                         ret = -1;
         glusterd_op_lock_ctx_t          *ctx = NULL;
         glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = NULL;
+        glusterd_conf_t                 *priv = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &unlock_req,
-                             (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req)) {
-                //failed to decode msg;
+        txn_id = &priv->global_txn_id;
+
+        ret = xdr_to_generic (req->msg[0], &unlock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "request received from peer");
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
 
-        gf_log ("glusterd", GF_LOG_INFO,
+        gf_log (this->name, GF_LOG_DEBUG,
                 "Received UNLOCK from uuid: %s", uuid_utoa (unlock_req.uuid));
 
         if (glusterd_friend_find_by_uuid (unlock_req.uuid, &peerinfo)) {
-                gf_log (THIS->name, GF_LOG_WARNING, "%s doesn't "
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
                         "belong to the cluster. Ignoring request.",
                         uuid_utoa (unlock_req.uuid));
                 ret = -1;
@@ -1370,8 +2340,9 @@ glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
         }
         uuid_copy (ctx->uuid, unlock_req.uuid);
         ctx->req = req;
+        ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, txn_id, ctx);
 
 out:
         glusterd_friend_sm ();
@@ -1381,14 +2352,25 @@ out:
 }
 
 int
+glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cluster_unlock);
+}
+
+int
 glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
                              int32_t op, int32_t status,
                              char *op_errstr, dict_t *rsp_dict)
 {
         gd1_mgmt_stage_op_rsp           rsp      = {{0},};
         int                             ret      = -1;
+        xlator_t                       *this     = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
+
         rsp.op_ret = status;
         glusterd_get_uuid (&rsp.uuid);
         rsp.op = op;
@@ -1397,11 +2379,10 @@ glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
         else
                 rsp.op_errstr = "";
 
-        ret = dict_allocate_and_serialize (rsp_dict,
-                                           &rsp.dict.dict_val,
-                                           (size_t *)&rsp.dict.dict_len);
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
         if (ret < 0) {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_ERROR,
                         "failed to get serialized length of dict");
                 return ret;
         }
@@ -1409,10 +2390,8 @@ glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to stage, ret: %d", ret);
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to stage, ret: %d", ret);
+        GF_FREE (rsp.dict.dict_val);
 
         return ret;
 }
@@ -1424,7 +2403,10 @@ glusterd_op_commit_send_resp (rpcsvc_request_t *req,
 {
         gd1_mgmt_commit_op_rsp          rsp      = {{0}, };
         int                             ret      = -1;
+        xlator_t                        *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
         rsp.op_ret = status;
         glusterd_get_uuid (&rsp.uuid);
@@ -1436,11 +2418,10 @@ glusterd_op_commit_send_resp (rpcsvc_request_t *req,
                 rsp.op_errstr = "";
 
         if (rsp_dict) {
-                ret = dict_allocate_and_serialize (rsp_dict,
-                                                   &rsp.dict.dict_val,
-                                                   (size_t *)&rsp.dict.dict_len);
+                ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                                   &rsp.dict.dict_len);
                 if (ret < 0) {
-                        gf_log ("", GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to get serialized length of dict");
                         goto out;
                 }
@@ -1450,24 +2431,24 @@ glusterd_op_commit_send_resp (rpcsvc_request_t *req,
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to commit, ret: %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to commit, ret: %d", ret);
 
 out:
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
+        GF_FREE (rsp.dict.dict_val);
         return ret;
 }
 
 int
-glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+__glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
 {
         int32_t                 ret = -1;
         gd1_mgmt_friend_req     friend_req = {{0},};
         gf_boolean_t            run_fsm = _gf_true;
 
         GF_ASSERT (req);
-        if (!xdr_to_generic (req->msg[0], &friend_req, (xdrproc_t)xdr_gd1_mgmt_friend_req)) {
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1486,8 +2467,7 @@ glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
         }
 
 out:
-        if (friend_req.hostname)
-                free (friend_req.hostname);//malloced by xdr
+        free (friend_req.hostname);//malloced by xdr
 
         if (run_fsm) {
                 glusterd_friend_sm ();
@@ -1498,14 +2478,23 @@ out:
 }
 
 int
-glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_incoming_friend_req);
+}
+
+int
+__glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
 {
         int32_t                 ret = -1;
         gd1_mgmt_friend_req     friend_req = {{0},};
         char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
 
         GF_ASSERT (req);
-        if (!xdr_to_generic (req->msg[0], &friend_req, (xdrproc_t)xdr_gd1_mgmt_friend_req)) {
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1524,10 +2513,8 @@ glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
                                             remote_hostname, friend_req.port);
 
 out:
-        if (friend_req.hostname)
-                free (friend_req.hostname);//malloced by xdr
-        if (friend_req.vols.vols_val)
-                free (friend_req.vols.vols_val);//malloced by xdr
+        free (friend_req.hostname);//malloced by xdr
+        free (friend_req.vols.vols_val);//malloced by xdr
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -1536,6 +2523,14 @@ out:
 }
 
 int
+glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_incoming_unfriend_req);
+
+}
+
+int
 glusterd_handle_friend_update_delete (dict_t *dict)
 {
         char                    *hostname = NULL;
@@ -1581,7 +2576,7 @@ out:
 }
 
 int
-glusterd_handle_friend_update (rpcsvc_request_t *req)
+__glusterd_handle_friend_update (rpcsvc_request_t *req)
 {
         int32_t                 ret = -1;
         gd1_mgmt_friend_update     friend_req = {{0},};
@@ -1607,7 +2602,9 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
         priv = this->private;
         GF_ASSERT (priv);
 
-        if (!xdr_to_generic (req->msg[0], &friend_req, (xdrproc_t)xdr_gd1_mgmt_friend_update)) {
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_update);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1667,7 +2664,13 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
                 gf_log ("", GF_LOG_INFO, "Received uuid: %s, hostname:%s",
                                 uuid_buf, hostname);
 
-                if (!uuid_compare (uuid, priv->uuid)) {
+                if (uuid_is_null (uuid)) {
+                        gf_log (this->name, GF_LOG_WARNING, "Updates mustn't "
+                                "contain peer with 'null' uuid");
+                        continue;
+                }
+
+                if (!uuid_compare (uuid, MY_UUID)) {
                         gf_log ("", GF_LOG_INFO, "Received my uuid as Friend");
                         i++;
                         continue;
@@ -1692,7 +2695,7 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
         }
 
 out:
-        uuid_copy (rsp.uuid, priv->uuid);
+        uuid_copy (rsp.uuid, MY_UUID);
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
         if (dict) {
@@ -1700,8 +2703,7 @@ out:
                         free (friend_req.friends.friends_val);//malloced by xdr
                 dict_unref (dict);
         } else {
-                if (friend_req.friends.friends_val)
-                        free (friend_req.friends.friends_val);//malloced by xdr
+                free (friend_req.friends.friends_val);//malloced by xdr
         }
 
         glusterd_friend_sm ();
@@ -1711,7 +2713,14 @@ out:
 }
 
 int
-glusterd_handle_probe_query (rpcsvc_request_t *req)
+glusterd_handle_friend_update (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_friend_update);
+}
+
+int
+__glusterd_handle_probe_query (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         xlator_t                        *this = NULL;
@@ -1725,7 +2734,9 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &probe_req, (xdrproc_t)xdr_gd1_mgmt_probe_req)) {
+        ret = xdr_to_generic (req->msg[0], &probe_req,
+                              (xdrproc_t)xdr_gd1_mgmt_probe_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1742,6 +2753,20 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
         gf_log ("glusterd", GF_LOG_INFO,
                 "Received probe from uuid: %s", uuid_utoa (probe_req.uuid));
 
+        /* Check for uuid collision and handle it in a user friendly way by
+         * sending the error.
+         */
+        if (!uuid_compare (probe_req.uuid, MY_UUID)) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Peer uuid %s is same as "
+                        "local uuid. Please check the uuid of both the peers "
+                        "from %s/%s", uuid_utoa (probe_req.uuid),
+                        GLUSTERD_DEFAULT_WORKDIR, GLUSTERD_INFO_FILE);
+                rsp.op_ret = -1;
+                rsp.op_errno = GF_PROBE_SAME_UUID;
+                rsp.port = port;
+                goto respond;
+        }
+
         ret = glusterd_remote_hostname_get (req, remote_hostname,
                                             sizeof (remote_hostname));
         if (ret) {
@@ -1766,20 +2791,22 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
                 }
         }
 
-        uuid_copy (rsp.uuid, conf->uuid);
+respond:
+        uuid_copy (rsp.uuid, MY_UUID);
 
         rsp.hostname = probe_req.hostname;
+        rsp.op_errstr = "";
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+        ret = 0;
 
         gf_log ("glusterd", GF_LOG_INFO, "Responded to %s, op_ret: %d, "
-                "op_errno: %d, ret: %d", probe_req.hostname,
+                "op_errno: %d, ret: %d", remote_hostname,
                 rsp.op_ret, rsp.op_errno, ret);
 
 out:
-        if (probe_req.hostname)
-                free (probe_req.hostname);//malloced by xdr
+        free (probe_req.hostname);//malloced by xdr
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -1787,8 +2814,13 @@ out:
         return ret;
 }
 
+int glusterd_handle_probe_query (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_probe_query);
+}
+
 int
-glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret     = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -1796,10 +2828,15 @@ glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
         glusterd_op_t                   cli_op = GD_OP_PROFILE_VOLUME;
         char                            *volname = NULL;
         int32_t                         op = 0;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -1815,42 +2852,50 @@ glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_log (THIS->name, GF_LOG_INFO, "Received volume profile req "
+        gf_log (this->name, GF_LOG_INFO, "Received volume profile req "
                 "for volume %s", volname);
         ret = dict_get_int32 (dict, "op", &op);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get op");
+                snprintf (err_str, sizeof (err_str), "Unable to get operation");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_cmd_log ("Volume stats", "volume  : %s, op: %d", volname, op);
-        ret = glusterd_op_begin (req, cli_op, dict);
-        gf_cmd_log ("Volume stats", " on volume %s, op: %d %s ",
-                    volname, op,
-                    ((ret == 0)? " SUCCEDED":" FAILED"));
+        ret = glusterd_op_begin (req, cli_op, dict, err_str, sizeof (err_str));
 
 out:
         glusterd_friend_sm ();
         glusterd_op_sm ();
 
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
-        if (ret)
+        free (cli_req.dict.dict_val);
+
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
+        }
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd_handle_getwd (rpcsvc_request_t *req)
+glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_profile_volume);
+}
+
+int
+__glusterd_handle_getwd (rpcsvc_request_t *req)
 {
         int32_t                 ret = -1;
         gf1_cli_getwd_rsp     rsp = {0,};
@@ -1865,8 +2910,9 @@ glusterd_handle_getwd (rpcsvc_request_t *req)
 
         rsp.wd = priv->workdir;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+        ret = 0;
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -1874,18 +2920,27 @@ glusterd_handle_getwd (rpcsvc_request_t *req)
         return ret;
 }
 
+int
+glusterd_handle_getwd (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_getwd);
+}
 
 int
-glusterd_handle_mount (rpcsvc_request_t *req)
+__glusterd_handle_mount (rpcsvc_request_t *req)
 {
         gf1_cli_mount_req mnt_req = {0,};
         gf1_cli_mount_rsp rsp     = {0,};
         dict_t *dict              = NULL;
         int ret                   = 0;
+        glusterd_conf_t     *priv   = NULL;
 
         GF_ASSERT (req);
+	priv = THIS->private;
 
-        if (!xdr_to_generic (req->msg[0], &mnt_req, (xdrproc_t)xdr_gf1_cli_mount_req)) {
+        ret = xdr_to_generic (req->msg[0], &mnt_req,
+                              (xdrproc_t)xdr_gf1_cli_mount_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 rsp.op_ret = -1;
@@ -1914,15 +2969,18 @@ glusterd_handle_mount (rpcsvc_request_t *req)
                 }
         }
 
+	synclock_unlock (&priv->big_lock);
         rsp.op_ret = glusterd_do_mount (mnt_req.label, dict,
                                         &rsp.path, &rsp.op_errno);
+	synclock_lock (&priv->big_lock);
 
  out:
         if (!rsp.path)
                 rsp.path = "";
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_mount_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_mount_rsp);
+        ret = 0;
 
         if (dict)
                 dict_unref (dict);
@@ -1936,7 +2994,13 @@ glusterd_handle_mount (rpcsvc_request_t *req)
 }
 
 int
-glusterd_handle_umount (rpcsvc_request_t *req)
+glusterd_handle_mount (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_mount);
+}
+
+int
+__glusterd_handle_umount (rpcsvc_request_t *req)
 {
         gf1_cli_umount_req umnt_req = {0,};
         gf1_cli_umount_rsp rsp      = {0,};
@@ -1949,11 +3013,15 @@ glusterd_handle_umount (rpcsvc_request_t *req)
         gf_boolean_t dir_ok         = _gf_false;
         char *pdir                  = NULL;
         char *t                     = NULL;
+        glusterd_conf_t     *priv   = NULL;
 
         GF_ASSERT (req);
         GF_ASSERT (this);
+        priv = this->private;
 
-        if (!xdr_to_generic (req->msg[0], &umnt_req, (xdrproc_t)xdr_gf1_cli_umount_req)) {
+        ret = xdr_to_generic (req->msg[0], &umnt_req,
+                              (xdrproc_t)xdr_gf1_cli_umount_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 rsp.op_ret = -1;
@@ -1992,7 +3060,9 @@ glusterd_handle_umount (rpcsvc_request_t *req)
         runner_add_args (&runner, "umount", umnt_req.path, NULL);
         if (umnt_req.lazy)
                 runner_add_arg (&runner, "-l");
+        synclock_unlock (&priv->big_lock);
         rsp.op_ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (rsp.op_ret == 0) {
                 if (realpath (umnt_req.path, mntp))
                         rmdir (mntp);
@@ -2010,8 +3080,9 @@ glusterd_handle_umount (rpcsvc_request_t *req)
         if (rsp.op_errno)
                 rsp.op_ret = -1;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_umount_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_umount_rsp);
+        ret = 0;
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
@@ -2020,6 +3091,12 @@ glusterd_handle_umount (rpcsvc_request_t *req)
 }
 
 int
+glusterd_handle_umount (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_umount);
+}
+
+int
 glusterd_friend_remove (uuid_t uuid, char *hostname)
 {
         int                           ret = 0;
@@ -2070,7 +3147,7 @@ out:
                 }
         }
 
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
@@ -2093,8 +3170,8 @@ glusterd_transport_keepalive_options_get (int *interval, int *time)
 }
 
 int
-glusterd_transport_inet_keepalive_options_build (dict_t **options,
-                                                 const char *hostname, int port)
+glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
+                                       int port)
 {
         dict_t  *dict = NULL;
         int32_t interval = -1;
@@ -2106,10 +3183,25 @@ glusterd_transport_inet_keepalive_options_build (dict_t **options,
 
         if (!port)
                 port = GLUSTERD_DEFAULT_PORT;
+
+        /* Build default transport options */
         ret = rpc_transport_inet_options_build (&dict, hostname, port);
         if (ret)
                 goto out;
 
+        /* Set frame-timeout to 10mins. Default timeout of 30 mins is too long
+         * when compared to 2 mins for cli timeout. This ensures users don't
+         * wait too long after cli timesout before being able to resume normal
+         * operations
+         */
+        ret = dict_set_int32 (dict, "frame-timeout", 600);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "Failed to set frame-timeout");
+                goto out;
+        }
+
+        /* Set keepalive options */
         glusterd_transport_keepalive_options_get (&interval, &time);
 
         if ((interval > 0) || (time > 0))
@@ -2121,65 +3213,117 @@ out:
 }
 
 int
-glusterd_friend_add (const char *hoststr, int port,
-                     glusterd_friend_sm_state_t state,
-                     uuid_t *uuid,
-                     glusterd_peerinfo_t **friend,
-                     gf_boolean_t restore,
-                     glusterd_peerctx_args_t *args)
+glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                            glusterd_peerctx_args_t *args)
 {
-        int                     ret = 0;
-        glusterd_conf_t        *conf = NULL;
-        glusterd_peerctx_t     *peerctx = NULL;
         dict_t                 *options = NULL;
-        gf_boolean_t            handover = _gf_false;
-
-        conf = THIS->private;
-        GF_ASSERT (conf);
-        GF_ASSERT (hoststr);
+        int                    ret = -1;
+        glusterd_peerctx_t     *peerctx = NULL;
+        data_t                 *data = NULL;
 
         peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
-        if (!peerctx) {
-                ret = -1;
+        if (!peerctx)
                 goto out;
-        }
 
         if (args)
                 peerctx->args = *args;
 
-        ret = glusterd_peerinfo_new (friend, state, uuid, hoststr);
+        peerctx->peerinfo = peerinfo;
+
+        ret = glusterd_transport_inet_options_build (&options,
+                                                     peerinfo->hostname,
+                                                     peerinfo->port);
         if (ret)
                 goto out;
 
-        peerctx->peerinfo = *friend;
+        /*
+         * For simulated multi-node testing, we need to make sure that we
+         * create our RPC endpoint with the same address that the peer would
+         * use to reach us.
+         */
+        if (this->options) {
+                data = dict_get(this->options,"transport.socket.bind-address");
+                if (data) {
+                        ret = dict_set(options,
+                                       "transport.socket.source-addr",data);
+                }
+        }
 
-        ret = glusterd_transport_inet_keepalive_options_build (&options,
-                                                         hoststr, port);
-        if (ret)
+        ret = glusterd_rpc_create (&peerinfo->rpc, options,
+                                   glusterd_peer_rpc_notify, peerctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to create rpc for"
+                        " peer %s", peerinfo->hostname);
                 goto out;
+        }
+        peerctx = NULL;
+        ret = 0;
+out:
+        GF_FREE (peerctx);
+        return ret;
+}
+
+int
+glusterd_friend_add (const char *hoststr, int port,
+                     glusterd_friend_sm_state_t state,
+                     uuid_t *uuid,
+                     glusterd_peerinfo_t **friend,
+                     gf_boolean_t restore,
+                     glusterd_peerctx_args_t *args)
+{
+        int                     ret = 0;
+        xlator_t               *this = NULL;
+        glusterd_conf_t        *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+        GF_ASSERT (hoststr);
 
-        ret = glusterd_rpc_create (&(*friend)->rpc, options,
-                                   glusterd_peer_rpc_notify,
-                                   peerctx);
+        ret = glusterd_peerinfo_new (friend, state, uuid, hoststr, port);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "failed to create rpc for"
-                        " peer %s", (char*)hoststr);
                 goto out;
         }
-        handover = _gf_true;
 
-out:
-        if (ret && !handover) {
-                        (void) glusterd_friend_cleanup (*friend);
-                        *friend = NULL;
+        /*
+         * We can't add to the list after calling glusterd_friend_rpc_create,
+         * even if it succeeds, because by then the callback to take it back
+         * off and free might have happened already (notably in the case of an
+         * invalid peer name).  That would mean we're adding something that had
+         * just been free, and we're likely to crash later.
+         */
+        list_add_tail (&(*friend)->uuid_list, &conf->peers);
+
+        //restore needs to first create the list of peers, then create rpcs
+        //to keep track of quorum in race-free manner. In restore for each peer
+        //rpc-create calls rpc_notify when the friend-list is partially
+        //constructed, leading to wrong quorum calculations.
+        if (!restore) {
+                ret = glusterd_store_peerinfo (*friend);
+                if (ret == 0) {
+                        synclock_unlock (&conf->big_lock);
+                        ret = glusterd_friend_rpc_create (this, *friend, args);
+                        synclock_lock (&conf->big_lock);
+                }
+                else {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store peerinfo");
+                }
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "connect returned %d", ret);
+        if (ret) {
+                (void) glusterd_friend_cleanup (*friend);
+                *friend = NULL;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_INFO, "connect returned %d", ret);
         return ret;
 }
 
 int
-glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
+glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
+                      dict_t *dict)
 {
         int                             ret = -1;
         glusterd_peerinfo_t             *peerinfo = NULL;
@@ -2195,6 +3339,7 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
                         " for host: %s (%d)", hoststr, port);
                 args.mode = GD_MODE_ON;
                 args.req  = req;
+                args.dict = dict;
                 ret = glusterd_friend_add ((char *)hoststr, port,
                                            GD_FRIEND_STATE_DEFAULT,
                                            NULL, &peerinfo, 0, &args);
@@ -2215,11 +3360,12 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
                         event->peerinfo = peerinfo;
                         ret = glusterd_friend_sm_inject_event (event);
                         glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_SUCCESS,
-                                                      (char*)hoststr, port);
+                                                      NULL, (char*)hoststr,
+                                                      port, dict);
                 }
         } else {
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND,
-                                              (char*)hoststr, port);
+                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND, NULL,
+                                              (char*)hoststr, port, dict);
         }
 
 out:
@@ -2229,7 +3375,7 @@ out:
 
 int
 glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                        uuid_t uuid)
+                        uuid_t uuid, dict_t *dict)
 {
         int                             ret = -1;
         glusterd_peerinfo_t             *peerinfo = NULL;
@@ -2270,6 +3416,7 @@ glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
         ctx->hostname = gf_strdup (hoststr);
         ctx->port = port;
         ctx->req = req;
+        ctx->dict = dict;
 
         event->ctx = ctx;
 
@@ -2304,7 +3451,7 @@ glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int por
 
         conf = this->private;
 
-        uuid_copy (rsp.uuid, conf->uuid);
+        uuid_copy (rsp.uuid, MY_UUID);
         rsp.hostname = hostname;
         rsp.port = port;
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
@@ -2317,76 +3464,236 @@ glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int por
 
 
 int
-glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *hostname, int port,
-                               int32_t op_ret, int32_t op_errno)
+glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
+                               char *remote_hostname, int port, int32_t op_ret,
+                               int32_t op_errno)
 {
         gd1_mgmt_friend_rsp  rsp = {{0}, };
         int32_t              ret = -1;
         xlator_t             *this = NULL;
         glusterd_conf_t      *conf = NULL;
 
-        GF_ASSERT (hostname);
+        GF_ASSERT (myhostname);
 
         this = THIS;
         GF_ASSERT (this);
 
         conf = this->private;
 
-        uuid_copy (rsp.uuid, conf->uuid);
+        uuid_copy (rsp.uuid, MY_UUID);
         rsp.op_ret = op_ret;
         rsp.op_errno = op_errno;
-        rsp.hostname = gf_strdup (hostname);
+        rsp.hostname = gf_strdup (myhostname);
         rsp.port = port;
 
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
 
         gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to %s (%d), ret: %d", hostname, port, ret);
-        if (rsp.hostname)
-                GF_FREE (rsp.hostname);
+                "Responded to %s (%d), ret: %d", remote_hostname, port, ret);
+        GF_FREE (rsp.hostname);
         return ret;
 }
 
+static void
+set_probe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
+                     size_t len, char *hostname, int port)
+{
+        if ((op_errstr) && (strcmp (op_errstr, ""))) {
+                snprintf (errstr, len, "%s", op_errstr);
+                return;
+        }
+
+        if (!op_ret) {
+                switch (op_errno) {
+                        case GF_PROBE_LOCALHOST:
+                                snprintf (errstr, len, "Probe on localhost not "
+                                          "needed");
+                                break;
+
+                        case GF_PROBE_FRIEND:
+                                snprintf (errstr, len, "Host %s port %d already"
+                                          " in peer list", hostname, port);
+                                break;
+
+                        default:
+                                if (op_errno != 0)
+                                        snprintf (errstr, len, "Probe returned "
+                                                  "with unknown errno %d",
+                                                  op_errno);
+                                break;
+                }
+        } else {
+                switch (op_errno) {
+                        case GF_PROBE_ANOTHER_CLUSTER:
+                                snprintf (errstr, len, "%s is already part of "
+                                          "another cluster", hostname);
+                                break;
+
+                        case GF_PROBE_VOLUME_CONFLICT:
+                                snprintf (errstr, len, "Atleast one volume on "
+                                          "%s conflicts with existing volumes "
+                                          "in the cluster", hostname);
+                                break;
+
+                        case GF_PROBE_UNKNOWN_PEER:
+                                snprintf (errstr, len, "%s responded with "
+                                          "'unknown peer' error, this could "
+                                          "happen if %s doesn't have localhost "
+                                          "in its peer database", hostname,
+                                          hostname);
+                                break;
+
+                        case GF_PROBE_ADD_FAILED:
+                                snprintf (errstr, len, "Failed to add peer "
+                                          "information on %s", hostname);
+                                break;
+
+                        case GF_PROBE_SAME_UUID:
+                                snprintf (errstr, len, "Peer uuid (host %s) is "
+                                          "same as local uuid", hostname);
+                                break;
+
+                        case GF_PROBE_QUORUM_NOT_MET:
+                                snprintf (errstr, len, "Cluster quorum is not "
+                                          "met. Changing peers is not allowed "
+                                          "in this state");
+                                break;
+
+                        default:
+                                snprintf (errstr, len, "Probe returned with "
+                                          "unknown errno %d", op_errno);
+                                break;
+                }
+        }
+}
+
 int
 glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *hostname, int port)
+                              int32_t op_errno, char *op_errstr, char *hostname,
+                              int port, dict_t *dict)
 {
-        gf1_cli_probe_rsp    rsp = {0, };
+        gf_cli_rsp           rsp = {0,};
         int32_t              ret = -1;
+        char        errstr[2048] = {0,};
+        char            *cmd_str = NULL;
+        xlator_t           *this = THIS;
 
         GF_ASSERT (req);
+        GF_ASSERT (this);
+
+        (void) set_probe_error_str (op_ret, op_errno, op_errstr, errstr,
+                                    sizeof (errstr), hostname, port);
+
+        if (dict) {
+                ret = dict_get_str (dict, "cmd-str", &cmd_str);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "command string");
+        }
 
         rsp.op_ret = op_ret;
         rsp.op_errno = op_errno;
-        rsp.hostname = hostname;
-        rsp.port = port;
+        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+
+        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
+                    (op_ret) ? "FAILED" : "SUCCESS",
+                    (errstr[0] != '\0') ? ":" : " ",
+                    (errstr[0] != '\0') ? errstr : " ");
 
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_probe_rsp);
+                                     (xdrproc_t)xdr_gf_cli_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO, "Responded to CLI, ret: %d",ret);
+        if (dict)
+                dict_unref (dict);
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to CLI, ret: %d",ret);
 
         return ret;
 }
 
+static void
+set_deprobe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
+                       size_t len, char *hostname)
+{
+        if ((op_errstr) && (strcmp (op_errstr, ""))) {
+                snprintf (errstr, len, "%s", op_errstr);
+                return;
+        }
+
+        if (op_ret) {
+                switch (op_errno) {
+                        case GF_DEPROBE_LOCALHOST:
+                                snprintf (errstr, len, "%s is localhost",
+                                hostname);
+                                break;
+
+                        case GF_DEPROBE_NOT_FRIEND:
+                                snprintf (errstr, len, "%s is not part of "
+                                          "cluster", hostname);
+                                break;
+
+                        case GF_DEPROBE_BRICK_EXIST:
+                                snprintf (errstr, len, "Brick(s) with the peer "
+                                          "%s exist in cluster", hostname);
+                                break;
+
+                        case GF_DEPROBE_FRIEND_DOWN:
+                                snprintf (errstr, len, "One of the peers is "
+                                          "probably down. Check with "
+                                          "'peer status'");
+                                break;
+
+                        case GF_DEPROBE_QUORUM_NOT_MET:
+                                snprintf (errstr, len, "Cluster quorum is not "
+                                          "met. Changing peers is not allowed "
+                                          "in this state");
+                                break;
+
+                        default:
+                                snprintf (errstr, len, "Detach returned with "
+                                          "unknown errno %d", op_errno);
+                                break;
+
+                }
+        }
+}
+
+
 int
 glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                                int32_t op_errno, char *hostname)
+                                int32_t op_errno, char *op_errstr,
+                                char *hostname, dict_t *dict)
 {
-        gf1_cli_deprobe_rsp    rsp = {0, };
+        gf_cli_rsp             rsp = {0,};
         int32_t                ret = -1;
+        char              *cmd_str = NULL;
+        char          errstr[2048] = {0,};
 
         GF_ASSERT (req);
 
+        (void) set_deprobe_error_str (op_ret, op_errno, op_errstr, errstr,
+                                      sizeof (errstr), hostname);
+
+        if (dict) {
+                ret = dict_get_str (dict, "cmd-str", &cmd_str);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
+                                "command string");
+        }
+
         rsp.op_ret = op_ret;
         rsp.op_errno = op_errno;
-        rsp.hostname = hostname;
+        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+
+        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
+                    (op_ret) ? "FAILED" : "SUCCESS",
+                    (errstr[0] != '\0') ? ":" : " ",
+                    (errstr[0] != '\0') ? errstr : " ");
 
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_deprobe_rsp);
+                                     (xdrproc_t)xdr_gf_cli_rsp);
 
-        gf_log ("glusterd", GF_LOG_INFO, "Responded to CLI, ret: %d",ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to CLI, ret: %d",ret);
 
         return ret;
 }
@@ -2400,39 +3707,52 @@ glusterd_list_friends (rpcsvc_request_t *req, dict_t *dict, int32_t flags)
         int32_t                 count = 0;
         dict_t                  *friends = NULL;
         gf1_cli_peer_list_rsp   rsp = {0,};
+        char                    my_uuid_str[64] = {0,};
+        char                    key[256] = {0,};
 
         priv = THIS->private;
         GF_ASSERT (priv);
 
-        if (!list_empty (&priv->peers)) {
-                friends = dict_new ();
-                if (!friends) {
-                        gf_log ("", GF_LOG_WARNING, "Out of Memory");
-                        goto out;
-                }
-        } else {
-                ret = 0;
+        friends = dict_new ();
+        if (!friends) {
+                gf_log ("", GF_LOG_WARNING, "Out of Memory");
                 goto out;
         }
-
-        if (flags == GF_CLI_LIST_ALL) {
-                        list_for_each_entry (entry, &priv->peers, uuid_list) {
-                                count++;
-                                ret = glusterd_add_peer_detail_to_dict (entry,
+        if (!list_empty (&priv->peers)) {
+                list_for_each_entry (entry, &priv->peers, uuid_list) {
+                        count++;
+                        ret = glusterd_add_peer_detail_to_dict (entry,
                                                                 friends, count);
-                                if (ret)
-                                        goto out;
+                        if (ret)
+                                goto out;
+                }
+        }
 
-                        }
+        if (flags == GF_CLI_LIST_POOL_NODES) {
+                count++;
+                snprintf (key, 256, "friend%d.uuid", count);
+                uuid_utoa_r (MY_UUID, my_uuid_str);
+                ret = dict_set_str (friends, key, my_uuid_str);
+                if (ret)
+                        goto out;
 
-                        ret = dict_set_int32 (friends, "count", count);
+                snprintf (key, 256, "friend%d.hostname", count);
+                ret = dict_set_str (friends, key, "localhost");
+                if (ret)
+                        goto out;
 
-                        if (ret)
-                                goto out;
+                snprintf (key, 256, "friend%d.connected", count);
+                ret = dict_set_int32 (friends, key, 1);
+                if (ret)
+                        goto out;
         }
 
+        ret = dict_set_int32 (friends, "count", count);
+        if (ret)
+                goto out;
+
         ret = dict_allocate_and_serialize (friends, &rsp.friends.friends_val,
-                                           (size_t *)&rsp.friends.friends_len);
+                                           &rsp.friends.friends_len);
 
         if (ret)
                 goto out;
@@ -2445,10 +3765,10 @@ out:
 
         rsp.op_ret = ret;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
-        if (rsp.friends.friends_val)
-                GF_FREE (rsp.friends.friends_val);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+        ret = 0;
+        GF_FREE (rsp.friends.friends_val);
 
         return ret;
 }
@@ -2539,7 +3859,7 @@ respond:
         if (ret)
                 goto out;
         ret = dict_allocate_and_serialize (volumes, &rsp.dict.dict_val,
-                                           (size_t *)&rsp.dict.dict_len);
+                                           &rsp.dict.dict_len);
 
         if (ret)
                 goto out;
@@ -2549,19 +3869,19 @@ out:
         rsp.op_ret = ret;
 
         rsp.op_errstr = "";
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_cli_rsp);
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_cli_rsp);
+        ret = 0;
 
         if (volumes)
                 dict_unref (volumes);
 
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
+        GF_FREE (rsp.dict.dict_val);
         return ret;
 }
 
 int
-glusterd_handle_status_volume (rpcsvc_request_t *req)
+__glusterd_handle_status_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret     = -1;
         uint32_t                        cmd     = 0;
@@ -2569,11 +3889,18 @@ glusterd_handle_status_volume (rpcsvc_request_t *req)
         char                           *volname = 0;
         gf_cli_req                      cli_req = {{0,}};
         glusterd_op_t                   cli_op  = GD_OP_STATUS_VOLUME;
+        char                            err_str[2048] = {0,};
+        xlator_t                       *this = NULL;
+        glusterd_conf_t                *conf = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -2586,8 +3913,10 @@ glusterd_handle_status_volume (rpcsvc_request_t *req)
                 ret = dict_unserialize (cli_req.dict.dict_val,
                                         cli_req.dict.dict_len, &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "unserialize buffer");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
 
@@ -2600,48 +3929,65 @@ glusterd_handle_status_volume (rpcsvc_request_t *req)
         if (!(cmd & GF_CLI_STATUS_ALL)) {
                 ret = dict_get_str (dict, "volname", &volname);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "failed to get volname");
+                        snprintf (err_str, sizeof (err_str), "Unable to get "
+                                  "volume name");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         goto out;
                 }
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "Received status volume req "
-                        "for volume %s", volname);
+                gf_log (this->name, GF_LOG_INFO,
+                        "Received status volume req for volume %s", volname);
 
         }
+        if ((cmd & GF_CLI_STATUS_QUOTAD) &&
+            (conf->op_version == GD_OP_VERSION_MIN)) {
+                snprintf (err_str, sizeof (err_str), "The cluster is operating "
+                          "at version 1. Getting the status of quotad is not "
+                          "allowed in this state.");
+                ret = -1;
+                goto out;
+        }
 
-        ret = glusterd_op_begin (req, GD_OP_STATUS_VOLUME, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_STATUS_VOLUME, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
 
-        if (ret)
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
 
         return ret;
 }
 
 int
-glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+glusterd_handle_status_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_status_volume);
+}
+
+int
+__glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
         glusterd_op_t                   cli_op = GD_OP_CLEARLOCKS_VOLUME;
         char                            *volname = NULL;
         dict_t                          *dict = NULL;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = -1;
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -2653,60 +3999,106 @@ glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to unserialize req-buffer to"
                                 " dictionary");
+                        snprintf (err_str, sizeof (err_str), "unable to decode "
+                                  "the command");
                         goto out;
                 }
 
         } else {
                 ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR, "Empty cli request.");
+                gf_log (this->name, GF_LOG_ERROR, "Empty cli request.");
                 goto out;
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_log (THIS->name, GF_LOG_INFO, "Received clear-locks volume req "
+        gf_log (this->name, GF_LOG_INFO, "Received clear-locks volume req "
                 "for volume %s", volname);
 
-        ret = glusterd_op_begin (req, cli_op, dict);
-
-        gf_cmd_log ("clear-locks", "on volume %s %s", volname,
-                    ((0 == ret) ? "SUCCEEDED" : "FAILED"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_CLEARLOCKS_VOLUME, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        return ret;
+}
+
+int
+glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_clearlocks_volume);
+}
 
+static int
+get_brickinfo_from_brickid (char *brickid, glusterd_brickinfo_t **brickinfo)
+{
+        glusterd_volinfo_t      *volinfo    = NULL;
+        char                    *volid_str  = NULL;
+        char                    *brick      = NULL;
+        char                    *brickid_dup = NULL;
+        uuid_t                  volid       = {0};
+        int                     ret         = -1;
+
+        brickid_dup = gf_strdup (brickid);
+        if (!brickid_dup)
+                goto out;
+
+        volid_str = brickid_dup;
+        brick = strchr (brickid_dup, ':');
+        if (!volid_str || !brick)
+                goto out;
+
+        *brick = '\0';
+        brick++;
+        uuid_parse (volid_str, volid);
+        ret = glusterd_volinfo_find_by_volume_id (volid, &volinfo);
         if (ret)
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
+                goto out;
 
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                      brickinfo);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        GF_FREE (brickid_dup);
         return ret;
 }
 
 int
-glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event,
-                          void *data)
+__glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
 {
         xlator_t                *this = NULL;
         glusterd_conf_t         *conf = NULL;
         int                     ret = 0;
+        char                    *brickid = NULL;
         glusterd_brickinfo_t    *brickinfo = NULL;
 
-        brickinfo = mydata;
-        if (!brickinfo)
+        brickid = mydata;
+        if (!brickid)
+                return 0;
+
+        ret = get_brickinfo_from_brickid (brickid, &brickinfo);
+        if (ret)
                 return 0;
 
         this = THIS;
@@ -2716,17 +4108,25 @@ glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 
         switch (event) {
         case RPC_CLNT_CONNECT:
-                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
+                gf_log (this->name, GF_LOG_DEBUG, "Connected to %s:%s",
+                        brickinfo->hostname, brickinfo->path);
                 glusterd_set_brick_status (brickinfo, GF_BRICK_STARTED);
                 ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
                 break;
 
         case RPC_CLNT_DISCONNECT:
-                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT");
+                if (GF_BRICK_STARTED == brickinfo->status)
+                        gf_log (this->name, GF_LOG_INFO, "Disconnected from "
+                                "%s:%s", brickinfo->hostname, brickinfo->path);
+
                 glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
                 break;
 
+        case RPC_CLNT_DESTROY:
+                GF_FREE (mydata);
+                mydata = NULL;
+                break;
         default:
                 gf_log (this->name, GF_LOG_TRACE,
                         "got some other RPC event %d", event);
@@ -2737,12 +4137,20 @@ glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 }
 
 int
-glusterd_shd_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                         rpc_clnt_event_t event,
-                         void *data)
+glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                           rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_brick_rpc_notify);
+}
+
+int
+__glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                               rpc_clnt_event_t event, void *data)
 {
         xlator_t                *this = NULL;
         glusterd_conf_t         *conf = NULL;
+        char                    *server = NULL;
         int                     ret = 0;
 
         this = THIS;
@@ -2750,17 +4158,21 @@ glusterd_shd_rpc_notify (struct rpc_clnt *rpc, void *mydata,
         conf = this->private;
         GF_ASSERT (conf);
 
+        server = mydata;
+        if (!server)
+                return 0;
+
         switch (event) {
         case RPC_CLNT_CONNECT:
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
-                (void) glusterd_shd_set_running (_gf_true);
+                (void) glusterd_nodesvc_set_online_status (server, _gf_true);
                 ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
                 break;
 
         case RPC_CLNT_DISCONNECT:
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT");
-                (void) glusterd_shd_set_running (_gf_false);
+                (void) glusterd_nodesvc_set_online_status (server, _gf_false);
                 break;
 
         default:
@@ -2773,10 +4185,29 @@ glusterd_shd_rpc_notify (struct rpc_clnt *rpc, void *mydata,
 }
 
 int
-glusterd_friend_remove_notify (glusterd_peerinfo_t *peerinfo, rpcsvc_request_t *req)
+glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                             rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_nodesvc_rpc_notify);
+}
+
+int
+glusterd_friend_remove_notify (glusterd_peerctx_t *peerctx)
 {
-        int ret = -1;
-        glusterd_friend_sm_event_t *new_event = NULL;
+        int                             ret = -1;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+        glusterd_peerinfo_t             *peerinfo = peerctx->peerinfo;
+        rpcsvc_request_t                *req = peerctx->args.req;
+        char                            *errstr = peerctx->errstr;
+        dict_t                          *dict = NULL;
+
+        GF_ASSERT (peerctx);
+
+        peerinfo = peerctx->peerinfo;
+        req = peerctx->args.req;
+        dict = peerctx->args.dict;
+        errstr = peerctx->errstr;
 
         ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
                                             &new_event);
@@ -2788,12 +4219,12 @@ glusterd_friend_remove_notify (glusterd_peerinfo_t *peerinfo, rpcsvc_request_t *
                         goto out;
                 }
 
-                glusterd_xfer_cli_probe_resp (req, -1, ENOTCONN,
-                                              peerinfo->hostname, peerinfo->port);
+                glusterd_xfer_cli_probe_resp (req, -1, ENOTCONN, errstr,
+                                              peerinfo->hostname,
+                                              peerinfo->port, dict);
 
                 new_event->peerinfo = peerinfo;
                 ret = glusterd_friend_sm_inject_event (new_event);
-                glusterd_friend_sm ();
 
         } else {
                 gf_log ("glusterd", GF_LOG_ERROR,
@@ -2806,17 +4237,17 @@ out:
 }
 
 int
-glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event,
-                          void *data)
+__glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data)
 {
         xlator_t             *this        = NULL;
         glusterd_conf_t      *conf        = NULL;
         int                   ret         = 0;
         glusterd_peerinfo_t  *peerinfo    = NULL;
         glusterd_peerctx_t   *peerctx     = NULL;
-        uuid_t                owner       = {0,};
-        uuid_t               *peer_uuid   = NULL;
+        gf_boolean_t         quorum_action = _gf_false;
+        glusterd_volinfo_t   *volinfo     = NULL;
+        uuid_t               uuid;
 
         peerctx = mydata;
         if (!peerctx)
@@ -2831,17 +4262,9 @@ glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
         {
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
                 peerinfo->connected = 1;
-                ret = glusterd_store_peerinfo (peerinfo);
-                if (ret) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to store "
-                                "peerinfo");
-                        break;
-                }
-
-                list_add_tail (&peerinfo->uuid_list, &conf->peers);
+                peerinfo->quorum_action = _gf_true;
 
-                ret = glusterd_peer_handshake (this, rpc, peerctx);
+                ret = glusterd_peer_dump_version (this, rpc, peerctx);
                 if (ret)
                         gf_log ("", GF_LOG_ERROR, "glusterd handshake failed");
                 break;
@@ -2852,43 +4275,46 @@ glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT %d",
                         peerinfo->state.state);
 
-                peerinfo->connected = 0;
+                if (peerinfo->connected) {
+                        if (conf->op_version < GD_OP_VERSION_4) {
+                                glusterd_get_lock_owner (&uuid);
+                                if (!uuid_is_null (uuid) &&
+                                    !uuid_compare (peerinfo->uuid, uuid))
+                                        glusterd_unlock (peerinfo->uuid);
+                        } else {
+                                list_for_each_entry (volinfo, &conf->volumes,
+                                                     vol_list) {
+                                        ret = glusterd_volume_unlock
+                                                    (volinfo->volname,
+                                                     peerinfo->uuid);
+                                        if (ret)
+                                                gf_log (this->name,
+                                                        GF_LOG_TRACE,
+                                                        "Lock not released "
+                                                        "for %s",
+                                                         volinfo->volname);
+                                }
+                        }
 
-                /*
-                  local glusterd (thinks that it) is the owner of the cluster
-                  lock and 'fails' the operation on the first disconnect from
-                  a peer.
-                */
-                glusterd_get_lock_owner (&owner);
-                if (!uuid_compare (conf->uuid, owner)) {
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_START_UNLOCK,
-                                                           NULL);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR, "Unable"
-                                        " to enqueue cluster unlock event");
-                        break;
+                        ret = 0;
                 }
 
-                peer_uuid = GF_CALLOC (1, sizeof (*peer_uuid), gf_common_mt_char);
-                if (!peer_uuid) {
-                        ret = -1;
-                        break;
+                if ((peerinfo->quorum_contrib != QUORUM_DOWN) &&
+                    (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
+                        peerinfo->quorum_contrib = QUORUM_DOWN;
+                        quorum_action = _gf_true;
+                        peerinfo->quorum_action = _gf_false;
                 }
 
-                uuid_copy (*peer_uuid, peerinfo->uuid);
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP,
-                                                   peer_uuid);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Unable"
-                                " to enque local lock flush event.");
-
-                //Inject friend disconnected here
-                if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT)  {
-                        glusterd_friend_remove_notify (peerinfo,
-                                                       peerctx->args.req);
+                /* Remove peer if it is not a friend and connection/handshake
+                *  fails, and notify cli. Happens only during probe.
+                */
+                if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT) {
+                        glusterd_friend_remove_notify (peerctx);
+                        goto out;
                 }
 
-                //default_notify (this, GF_EVENT_CHILD_DOWN, NULL);
+                peerinfo->connected = 0;
                 break;
         }
         default:
@@ -2898,12 +4324,23 @@ glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
                 break;
         }
 
+out:
         glusterd_friend_sm ();
         glusterd_op_sm ();
+        if (quorum_action)
+                glusterd_do_quorum_action ();
         return ret;
 }
 
 int
+glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_peer_rpc_notify);
+}
+
+int
 glusterd_null (rpcsvc_request_t *req)
 {
 
@@ -2911,11 +4348,11 @@ glusterd_null (rpcsvc_request_t *req)
 }
 
 rpcsvc_actor_t gd_svc_mgmt_actors[] = {
-        [GLUSTERD_MGMT_NULL]           = { "NULL", GLUSTERD_MGMT_NULL, glusterd_null, NULL, NULL, 0},
-        [GLUSTERD_MGMT_CLUSTER_LOCK]   = { "CLUSTER_LOCK", GLUSTERD_MGMT_CLUSTER_LOCK, glusterd_handle_cluster_lock, NULL, NULL, 0},
-        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = { "CLUSTER_UNLOCK", GLUSTERD_MGMT_CLUSTER_UNLOCK, glusterd_handle_cluster_unlock, NULL, NULL, 0},
-        [GLUSTERD_MGMT_STAGE_OP]       = { "STAGE_OP", GLUSTERD_MGMT_STAGE_OP, glusterd_handle_stage_op, NULL, NULL, 0},
-        [GLUSTERD_MGMT_COMMIT_OP]      = { "COMMIT_OP", GLUSTERD_MGMT_COMMIT_OP, glusterd_handle_commit_op, NULL, NULL, 0},
+        [GLUSTERD_MGMT_NULL]           = { "NULL",           GLUSTERD_MGMT_NULL,           glusterd_null,                  NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_CLUSTER_LOCK]   = { "CLUSTER_LOCK",   GLUSTERD_MGMT_CLUSTER_LOCK,   glusterd_handle_cluster_lock,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = { "CLUSTER_UNLOCK", GLUSTERD_MGMT_CLUSTER_UNLOCK, glusterd_handle_cluster_unlock, NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_STAGE_OP]       = { "STAGE_OP",       GLUSTERD_MGMT_STAGE_OP,       glusterd_handle_stage_op,       NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_COMMIT_OP]      = { "COMMIT_OP",      GLUSTERD_MGMT_COMMIT_OP,      glusterd_handle_commit_op,      NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program gd_svc_mgmt_prog = {
@@ -2924,14 +4361,15 @@ struct rpcsvc_program gd_svc_mgmt_prog = {
         .progver   = GD_MGMT_VERSION,
         .numactors = GLUSTERD_MGMT_MAXVALUE,
         .actors    = gd_svc_mgmt_actors,
+	.synctask  = _gf_true,
 };
 
 rpcsvc_actor_t gd_svc_peer_actors[] = {
-        [GLUSTERD_FRIEND_NULL]    = { "NULL", GLUSTERD_MGMT_NULL, glusterd_null, NULL, NULL, 0},
-        [GLUSTERD_PROBE_QUERY]    = { "PROBE_QUERY", GLUSTERD_PROBE_QUERY, glusterd_handle_probe_query, NULL, NULL, 0},
-        [GLUSTERD_FRIEND_ADD]     = { "FRIEND_ADD", GLUSTERD_FRIEND_ADD, glusterd_handle_incoming_friend_req, NULL, NULL, 0},
-        [GLUSTERD_FRIEND_REMOVE]  = { "FRIEND_REMOVE", GLUSTERD_FRIEND_REMOVE, glusterd_handle_incoming_unfriend_req, NULL, NULL, 0},
-        [GLUSTERD_FRIEND_UPDATE]  = { "FRIEND_UPDATE", GLUSTERD_FRIEND_UPDATE, glusterd_handle_friend_update, NULL, NULL, 0},
+        [GLUSTERD_FRIEND_NULL]    = { "NULL",          GLUSTERD_MGMT_NULL,     glusterd_null,                         NULL, 0, DRC_NA},
+        [GLUSTERD_PROBE_QUERY]    = { "PROBE_QUERY",   GLUSTERD_PROBE_QUERY,   glusterd_handle_probe_query,           NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_ADD]     = { "FRIEND_ADD",    GLUSTERD_FRIEND_ADD,    glusterd_handle_incoming_friend_req,   NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_REMOVE]  = { "FRIEND_REMOVE", GLUSTERD_FRIEND_REMOVE, glusterd_handle_incoming_unfriend_req, NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_UPDATE]  = { "FRIEND_UPDATE", GLUSTERD_FRIEND_UPDATE, glusterd_handle_friend_update,         NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program gd_svc_peer_prog = {
@@ -2940,39 +4378,44 @@ struct rpcsvc_program gd_svc_peer_prog = {
         .progver   = GD_FRIEND_VERSION,
         .numactors = GLUSTERD_FRIEND_MAXVALUE,
         .actors    = gd_svc_peer_actors,
+	.synctask  = _gf_false,
 };
 
 
 
 rpcsvc_actor_t gd_svc_cli_actors[] = {
-        [GLUSTER_CLI_PROBE]         = { "CLI_PROBE", GLUSTER_CLI_PROBE, glusterd_handle_cli_probe, NULL, NULL, 0},
-        [GLUSTER_CLI_CREATE_VOLUME] = { "CLI_CREATE_VOLUME", GLUSTER_CLI_CREATE_VOLUME, glusterd_handle_create_volume, NULL,NULL, 0},
-        [GLUSTER_CLI_DEFRAG_VOLUME] = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME, glusterd_handle_defrag_volume, NULL,NULL, 0},
-        [GLUSTER_CLI_DEPROBE]       = { "FRIEND_REMOVE", GLUSTER_CLI_DEPROBE, glusterd_handle_cli_deprobe, NULL, NULL, 0},
-        [GLUSTER_CLI_LIST_FRIENDS]  = { "LIST_FRIENDS", GLUSTER_CLI_LIST_FRIENDS, glusterd_handle_cli_list_friends, NULL, NULL, 0},
-        [GLUSTER_CLI_START_VOLUME]  = { "START_VOLUME", GLUSTER_CLI_START_VOLUME, glusterd_handle_cli_start_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_STOP_VOLUME]   = { "STOP_VOLUME", GLUSTER_CLI_STOP_VOLUME, glusterd_handle_cli_stop_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_DELETE_VOLUME] = { "DELETE_VOLUME", GLUSTER_CLI_DELETE_VOLUME, glusterd_handle_cli_delete_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_GET_VOLUME]    = { "GET_VOLUME", GLUSTER_CLI_GET_VOLUME, glusterd_handle_cli_get_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_ADD_BRICK]     = { "ADD_BRICK", GLUSTER_CLI_ADD_BRICK, glusterd_handle_add_brick, NULL, NULL, 0},
-        [GLUSTER_CLI_REPLACE_BRICK] = { "REPLACE_BRICK", GLUSTER_CLI_REPLACE_BRICK, glusterd_handle_replace_brick, NULL, NULL, 0},
-        [GLUSTER_CLI_REMOVE_BRICK]  = { "REMOVE_BRICK", GLUSTER_CLI_REMOVE_BRICK, glusterd_handle_remove_brick, NULL, NULL, 0},
-        [GLUSTER_CLI_LOG_ROTATE]    = { "LOG FILENAME", GLUSTER_CLI_LOG_ROTATE, glusterd_handle_log_rotate, NULL, NULL, 0},
-        [GLUSTER_CLI_SET_VOLUME]    = { "SET_VOLUME", GLUSTER_CLI_SET_VOLUME, glusterd_handle_set_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_SYNC_VOLUME]   = { "SYNC_VOLUME", GLUSTER_CLI_SYNC_VOLUME, glusterd_handle_sync_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_RESET_VOLUME]  = { "RESET_VOLUME", GLUSTER_CLI_RESET_VOLUME, glusterd_handle_reset_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_FSM_LOG]       = { "FSM_LOG", GLUSTER_CLI_FSM_LOG, glusterd_handle_fsm_log, NULL, NULL, 0},
-        [GLUSTER_CLI_GSYNC_SET]     = { "GSYNC_SET", GLUSTER_CLI_GSYNC_SET, glusterd_handle_gsync_set, NULL, NULL, 0},
-        [GLUSTER_CLI_PROFILE_VOLUME] = { "STATS_VOLUME", GLUSTER_CLI_PROFILE_VOLUME, glusterd_handle_cli_profile_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_QUOTA]         = { "QUOTA", GLUSTER_CLI_QUOTA, glusterd_handle_quota, NULL, NULL, 0},
-        [GLUSTER_CLI_GETWD]         = { "GETWD", GLUSTER_CLI_GETWD, glusterd_handle_getwd, NULL, NULL, 1},
-        [GLUSTER_CLI_STATUS_VOLUME]  = {"STATUS_VOLUME", GLUSTER_CLI_STATUS_VOLUME, glusterd_handle_status_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_MOUNT]         = { "MOUNT", GLUSTER_CLI_MOUNT, glusterd_handle_mount, NULL, NULL, 1},
-        [GLUSTER_CLI_UMOUNT]        = { "UMOUNT", GLUSTER_CLI_UMOUNT, glusterd_handle_umount, NULL, NULL, 1},
-        [GLUSTER_CLI_HEAL_VOLUME]  = { "HEAL_VOLUME", GLUSTER_CLI_HEAL_VOLUME, glusterd_handle_cli_heal_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME", GLUSTER_CLI_STATEDUMP_VOLUME, glusterd_handle_cli_statedump_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_LIST_VOLUME] = {"LIST_VOLUME", GLUSTER_CLI_LIST_VOLUME, glusterd_handle_cli_list_volume, NULL, NULL, 0},
-        [GLUSTER_CLI_CLRLOCKS_VOLUME] = {"CLEARLOCKS_VOLUME", GLUSTER_CLI_CLRLOCKS_VOLUME, glusterd_handle_cli_clearlocks_volume, NULL, NULL, 0},
+        [GLUSTER_CLI_PROBE]              = { "CLI_PROBE",         GLUSTER_CLI_PROBE,            glusterd_handle_cli_probe,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_CREATE_VOLUME]      = { "CLI_CREATE_VOLUME", GLUSTER_CLI_CREATE_VOLUME,    glusterd_handle_create_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DEFRAG_VOLUME]      = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME,    glusterd_handle_defrag_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DEPROBE]            = { "FRIEND_REMOVE",     GLUSTER_CLI_DEPROBE,          glusterd_handle_cli_deprobe,           NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_RESET]         = { "UUID_RESET",        GLUSTER_CLI_UUID_RESET,       glusterd_handle_cli_uuid_reset,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_START_VOLUME]       = { "START_VOLUME",      GLUSTER_CLI_START_VOLUME,     glusterd_handle_cli_start_volume,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_STOP_VOLUME]        = { "STOP_VOLUME",       GLUSTER_CLI_STOP_VOLUME,      glusterd_handle_cli_stop_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DELETE_VOLUME]      = { "DELETE_VOLUME",     GLUSTER_CLI_DELETE_VOLUME,    glusterd_handle_cli_delete_volume,     NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_ADD_BRICK]          = { "ADD_BRICK",         GLUSTER_CLI_ADD_BRICK,        glusterd_handle_add_brick,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_REPLACE_BRICK]      = { "REPLACE_BRICK",     GLUSTER_CLI_REPLACE_BRICK,    glusterd_handle_replace_brick,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_REMOVE_BRICK]       = { "REMOVE_BRICK",      GLUSTER_CLI_REMOVE_BRICK,     glusterd_handle_remove_brick,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LOG_ROTATE]         = { "LOG FILENAME",      GLUSTER_CLI_LOG_ROTATE,       glusterd_handle_log_rotate,            NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SET_VOLUME]         = { "SET_VOLUME",        GLUSTER_CLI_SET_VOLUME,       glusterd_handle_set_volume,            NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SYNC_VOLUME]        = { "SYNC_VOLUME",       GLUSTER_CLI_SYNC_VOLUME,      glusterd_handle_sync_volume,           NULL, 0, DRC_NA},
+        [GLUSTER_CLI_RESET_VOLUME]       = { "RESET_VOLUME",      GLUSTER_CLI_RESET_VOLUME,     glusterd_handle_reset_volume,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_FSM_LOG]            = { "FSM_LOG",           GLUSTER_CLI_FSM_LOG,          glusterd_handle_fsm_log,               NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GSYNC_SET]          = { "GSYNC_SET",         GLUSTER_CLI_GSYNC_SET,        glusterd_handle_gsync_set,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_PROFILE_VOLUME]     = { "STATS_VOLUME",      GLUSTER_CLI_PROFILE_VOLUME,   glusterd_handle_cli_profile_volume,    NULL, 0, DRC_NA},
+        [GLUSTER_CLI_QUOTA]              = { "QUOTA",             GLUSTER_CLI_QUOTA,            glusterd_handle_quota,                 NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_MOUNT]              = { "MOUNT",             GLUSTER_CLI_MOUNT,            glusterd_handle_mount,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_UMOUNT]             = { "UMOUNT",            GLUSTER_CLI_UMOUNT,           glusterd_handle_umount,                NULL, 1, DRC_NA},
+        [GLUSTER_CLI_HEAL_VOLUME]        = { "HEAL_VOLUME",       GLUSTER_CLI_HEAL_VOLUME,      glusterd_handle_cli_heal_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_STATEDUMP_VOLUME]   = {"STATEDUMP_VOLUME",   GLUSTER_CLI_STATEDUMP_VOLUME, glusterd_handle_cli_statedump_volume,  NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_CLRLOCKS_VOLUME]    = {"CLEARLOCKS_VOLUME",  GLUSTER_CLI_CLRLOCKS_VOLUME,  glusterd_handle_cli_clearlocks_volume, NULL, 0, DRC_NA},
+        [GLUSTER_CLI_COPY_FILE]     = {"COPY_FILE", GLUSTER_CLI_COPY_FILE, glusterd_handle_copy_file, NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SYS_EXEC]      = {"SYS_EXEC", GLUSTER_CLI_SYS_EXEC, glusterd_handle_sys_exec, NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program gd_svc_cli_prog = {
@@ -2981,4 +4424,41 @@ struct rpcsvc_program gd_svc_cli_prog = {
         .progver   = GLUSTER_CLI_VERSION,
         .numactors = GLUSTER_CLI_MAXVALUE,
         .actors    = gd_svc_cli_actors,
+	.synctask  = _gf_true,
+};
+
+/* This is a minimal RPC prog, which contains only the readonly RPC procs from
+ * the cli rpcsvc
+ */
+rpcsvc_actor_t gd_svc_cli_actors_ro[] = {
+        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_cli_prog_ro = {
+        .progname  = "GlusterD svc cli read-only",
+        .prognum   = GLUSTER_CLI_PROGRAM,
+        .progver   = GLUSTER_CLI_VERSION,
+        .numactors = GLUSTER_CLI_MAXVALUE,
+        .actors    = gd_svc_cli_actors_ro,
+	.synctask  = _gf_true,
+};
+
+rpcsvc_actor_t gd_svc_mgmt_v3_actors[] = {
+        [GLUSTERD_MGMT_V3_NULL]          = { "NULL",       GLUSTERD_MGMT_V3_NULL,          glusterd_null,                 NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_VOLUME_LOCK]   = { "VOL_LOCK",   GLUSTERD_MGMT_V3_VOLUME_LOCK,   glusterd_handle_volume_lock,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_VOLUME_UNLOCK] = { "VOL_UNLOCK", GLUSTERD_MGMT_V3_VOLUME_UNLOCK, glusterd_handle_volume_unlock, NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_mgmt_v3_prog = {
+        .progname  = "GlusterD svc mgmt v3",
+        .prognum   = GD_MGMT_PROGRAM,
+        .progver   = GD_MGMT_V3_VERSION,
+        .numactors = GLUSTERD_MGMT_V3_MAXVALUE,
+        .actors    = gd_svc_mgmt_v3_actors,
+        .synctask  = _gf_true,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index 64945a716..e0508faf6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -31,6 +21,7 @@
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-op-sm.h"
+#include "glusterd-store.h"
 
 #include "glusterfs3.h"
 #include "protocol-common.h"
@@ -39,6 +30,7 @@
 
 extern struct rpc_clnt_program gd_peer_prog;
 extern struct rpc_clnt_program gd_mgmt_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
 
 #define TRUSTED_PREFIX         "trusted-"
 
@@ -113,19 +105,107 @@ build_volfile_path (const char *volname, char *path,
 
         ret = 1;
 out:
-        if (free_ptr)
-                GF_FREE (free_ptr);
+        GF_FREE (free_ptr);
+        return ret;
+}
+
+/* Get and store op-versions of the clients sending the getspec request
+ * Clients of versions <= 3.3, don't send op-versions, their op-versions are
+ * defaulted to 1
+ */
+static int
+_get_client_op_versions (gf_getspec_req *args, peer_info_t *peerinfo)
+{
+        int    ret                   = 0;
+        int    client_max_op_version = 1;
+        int    client_min_op_version = 1;
+        dict_t *dict                 = NULL;
+
+        GF_ASSERT (args);
+        GF_ASSERT (peerinfo);
+
+        if (args->xdata.xdata_len) {
+                dict = dict_new ();
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_unserialize (args->xdata.xdata_val,
+                                        args->xdata.xdata_len, &dict);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to unserialize request dictionary");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "min-op-version",
+                                      &client_min_op_version);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to get client-min-op-version");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "max-op-version",
+                                      &client_max_op_version);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to get client-max-op-version");
+                        goto out;
+                }
+        }
+
+        peerinfo->max_op_version = client_max_op_version;
+        peerinfo->min_op_version = client_min_op_version;
+
+out:
+        return ret;
+}
+
+/* Checks if the client supports the volume, ie. client can understand all the
+ * options in the volfile
+ */
+static gf_boolean_t
+_client_supports_volume (peer_info_t *peerinfo, int32_t *op_errno)
+{
+        gf_boolean_t       ret       = _gf_true;
+        glusterd_volinfo_t *volinfo  = NULL;
+
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (op_errno);
+
+
+        /* Only check when the volfile being requested is a volume. Not finding
+         * a volinfo implies that the volfile requested for is not of a gluster
+         * volume. A non volume volfile is requested by the local gluster
+         * services like shd and nfs-server. These need not be checked as they
+         * will be running at the same op-version as glusterd and will be able
+         * to support all the features
+         */
+        if ((glusterd_volinfo_find (peerinfo->volname, &volinfo) == 0) &&
+            ((peerinfo->min_op_version > volinfo->client_op_version) ||
+             (peerinfo->max_op_version < volinfo->client_op_version))) {
+                ret = _gf_false;
+                *op_errno = ENOTSUP;
+                gf_log ("glusterd", GF_LOG_INFO,
+                        "Client %s (%d -> %d) doesn't support required "
+                        "op-version (%d). Rejecting volfile request.",
+                        peerinfo->identifier, peerinfo->min_op_version,
+                        peerinfo->max_op_version, volinfo->client_op_version);
+        }
+
         return ret;
 }
 
 int
-server_getspec (rpcsvc_request_t *req)
+__server_getspec (rpcsvc_request_t *req)
 {
         int32_t               ret                    = -1;
         int32_t               op_errno               = 0;
         int32_t               spec_fd                = -1;
         size_t                file_len               = 0;
-        char                  filename[ZR_PATH_MAX]  = {0,};
+        char                  filename[PATH_MAX]  = {0,};
         struct stat           stbuf                  = {0,};
         char                 *volume                 = NULL;
         char                 *tmp                    = NULL;
@@ -134,28 +214,53 @@ server_getspec (rpcsvc_request_t *req)
         gf_getspec_req        args                   = {0,};
         gf_getspec_rsp        rsp                    = {0,};
         char                  addrstr[RPCSVC_PEER_STRLEN] = {0};
+        peer_info_t          *peerinfo               = NULL;
 
-
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_gf_getspec_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_getspec_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
 
+        peerinfo = &req->trans->peerinfo;
+
         volume = args.key;
+        /* Need to strip leading '/' from volnames. This was introduced to
+         * support nfs style mount parameters for native gluster mount
+         */
+        if (volume[0] == '/')
+                strncpy (peerinfo->volname, &volume[1], strlen(&volume[1]));
+        else
+                strncpy (peerinfo->volname, volume, strlen(volume));
+
+        ret = _get_client_op_versions (&args, peerinfo);
+        if (ret)
+                goto fail;
+
+        if (!_client_supports_volume (peerinfo, &op_errno)) {
+                ret = -1;
+                goto fail;
+        }
 
         trans = req->trans;
+        /* addrstr will be empty for cli socket connections */
         ret = rpcsvc_transport_peername (trans, (char *)&addrstr,
                                          sizeof (addrstr));
         if (ret)
                 goto fail;
 
-        tmp = strrchr (addrstr, ':');
-        *tmp = '\0';
+        tmp  = strrchr (addrstr, ':');
+        if (tmp)
+                *tmp = '\0';
 
-        /* we trust the local admin */
-        if (!glusterd_is_local_addr (addrstr)) {
+        /* The trusted volfiles are given to the glusterd owned process like NFS
+         * server, self-heal daemon etc., so that they are not inadvertently
+         * blocked by a auth.{allow,reject} setting. The trusted volfile is not
+         * meant for external users.
+         */
+        if (strlen (addrstr) && gf_is_local_addr (addrstr)) {
 
                 ret = build_volfile_path (volume, filename,
                                           sizeof (filename),
@@ -209,45 +314,360 @@ fail:
                 rsp.op_errno = cookie;
 
         if (!rsp.spec)
-                rsp.spec = "";
+                rsp.spec = strdup ("");
 
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_gf_getspec_rsp);
-        if (args.key)
-                free (args.key);//malloced by xdr
-        if (rsp.spec && (strcmp (rsp.spec, "")))
-                free (rsp.spec);
+        free (args.key);//malloced by xdr
+        free (rsp.spec);
+
+        return 0;
+}
+
+int
+server_getspec (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __server_getspec);
+}
+
+int32_t
+__server_event_notify (rpcsvc_request_t *req)
+{
+        int32_t                 ret             = -1;
+        int32_t                 op_errno        =  0;
+        gf_event_notify_req     args            = {0,};
+        gf_event_notify_rsp     rsp             = {0,};
+        dict_t                 *dict            = NULL;
+        gf_boolean_t            need_rsp        = _gf_true;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_event_notify_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        if (args.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        return ret;
+                ret = dict_unserialize (args.dict.dict_val,
+                                        args.dict.dict_len, &dict);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to unserialize req");
+                        goto fail;
+                }
+        }
+
+        switch (args.op) {
+        case GF_EN_DEFRAG_STATUS:
+                gf_log ("", GF_LOG_INFO,
+                        "received defrag status updated");
+                if (dict) {
+                        glusterd_defrag_event_notify_handle (dict);
+                        need_rsp = _gf_false;
+                }
+                break;
+        default:
+                gf_log ("", GF_LOG_ERROR, "Unknown op received in event "
+                        "notify");
+                ret = -1;
+                break;
+        }
+
+fail:
+        rsp.op_ret   = ret;
+
+        if (op_errno)
+                rsp.op_errno = gf_errno_to_error (op_errno);
+
+        if (need_rsp)
+                glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                       (xdrproc_t)xdr_gf_event_notify_rsp);
+        if (dict)
+                dict_unref (dict);
+        free (args.dict.dict_val);//malloced by xdr
 
         return 0;
 }
 
+int32_t
+server_event_notify (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __server_event_notify);
+}
+
+int
+gd_validate_cluster_op_version (xlator_t *this, int cluster_op_version,
+                                char *peerid)
+{
+        int              ret  = -1;
+        glusterd_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (cluster_op_version > GD_OP_VERSION_MAX) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "operating version %d is more than the maximum "
+                        "supported (%d) on the machine (as per peer request "
+                        "from %s)", cluster_op_version, GD_OP_VERSION_MAX,
+                        peerid);
+                goto out;
+        }
+
+        /* The peer can only reduce its op-version when it doesn't have any
+         * volumes. Reducing op-version when it already contains volumes can
+         * lead to inconsistencies in the cluster
+         */
+        if ((cluster_op_version < conf->op_version) &&
+            !list_empty (&conf->volumes)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot reduce operating version to %d from current "
+                        "version %d as volumes exist (as per peer request from "
+                        "%s)", cluster_op_version, conf->op_version, peerid);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+__glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+{
+        dict_t            *dict            = NULL;
+        xlator_t          *this            = NULL;
+        glusterd_conf_t   *conf            = NULL;
+        int                ret             = -1;
+        int                op_errno        = EINVAL;
+        gf_mgmt_hndsk_req  args            = {{0,},};
+        gf_mgmt_hndsk_rsp  rsp             = {0,};
+
+        this = THIS;
+        conf = this->private;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_int32 (dict, GD_OP_VERSION_KEY, conf->op_version);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set operating version");
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, GD_MIN_OP_VERSION_KEY, GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s", GD_MIN_OP_VERSION_KEY);
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, GD_MAX_OP_VERSION_KEY, GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s", GD_MAX_OP_VERSION_KEY);
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, (&rsp.hndsk.hndsk_val),
+                                    rsp.hndsk.hndsk_len, op_errno, out);
+out:
+
+        rsp.op_ret = ret;
+        rsp.op_errno = op_errno;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+
+        ret = 0;
+
+        if (dict)
+                dict_unref (dict);
+
+        if (args.hndsk.hndsk_val)
+                free (args.hndsk.hndsk_val);
+
+        if (rsp.hndsk.hndsk_val)
+                GF_FREE (rsp.hndsk.hndsk_val);
+
+        return ret;
+}
+
+int
+glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_mgmt_hndsk_versions);
+}
+
+int
+__glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+{
+        dict_t            *clnt_dict       = NULL;
+        xlator_t          *this            = NULL;
+        glusterd_conf_t   *conf            = NULL;
+        int                ret             = -1;
+        int                op_errno        = EINVAL;
+        int                peer_op_version = 0;
+        gf_mgmt_hndsk_req  args            = {{0,},};
+        gf_mgmt_hndsk_rsp  rsp             = {0,};
+
+        this = THIS;
+        conf = this->private;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, clnt_dict, args.hndsk.hndsk_val,
+                                      (args.hndsk.hndsk_len), ret, op_errno,
+                                      out);
+
+        ret = dict_get_int32 (clnt_dict, GD_OP_VERSION_KEY, &peer_op_version);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to get the op-version key peer=%s",
+                        req->trans->peerinfo.identifier);
+                goto out;
+        }
+
+        ret = gd_validate_cluster_op_version (this, peer_op_version,
+                                              req->trans->peerinfo.identifier);
+        if (ret)
+                goto out;
+
+
+        /* As this is ACK from the Cluster for the versions supported,
+           can set the op-version of 'this' glusterd to the one
+           received. */
+        gf_log (this->name, GF_LOG_INFO, "using the op-version %d",
+                peer_op_version);
+        conf->op_version = peer_op_version;
+        ret = glusterd_store_global_info (this);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store op-version");
+
+out:
+        rsp.op_ret = ret;
+        rsp.op_errno = op_errno;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+
+        ret = 0;
+
+        if (clnt_dict)
+                dict_unref (clnt_dict);
+
+        if (args.hndsk.hndsk_val)
+                free (args.hndsk.hndsk_val);
+
+        return ret;
+}
+
+int
+glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_mgmt_hndsk_versions_ack);
+}
 
 rpcsvc_actor_t gluster_handshake_actors[] = {
-        [GF_HNDSK_NULL]      = {"NULL",      GF_HNDSK_NULL,      NULL, NULL, NULL, 0},
-        [GF_HNDSK_GETSPEC]   = {"GETSPEC",   GF_HNDSK_GETSPEC,   server_getspec, NULL, NULL, 0},
+        [GF_HNDSK_NULL]         = {"NULL",        GF_HNDSK_NULL,         NULL,                NULL, 0, DRC_NA},
+        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+        [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_HNDSK_EVENT_NOTIFY, server_event_notify, NULL, 0, DRC_NA},
 };
 
 
 struct rpcsvc_program gluster_handshake_prog = {
-        .progname  = "GlusterFS Handshake",
+        .progname  = "Gluster Handshake",
         .prognum   = GLUSTER_HNDSK_PROGRAM,
         .progver   = GLUSTER_HNDSK_VERSION,
         .actors    = gluster_handshake_actors,
         .numactors = GF_HNDSK_MAXVALUE,
 };
 
+/* A minimal RPC program just for the cli getspec command */
+rpcsvc_actor_t gluster_cli_getspec_actors[] = {
+        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gluster_cli_getspec_prog = {
+        .progname  = "Gluster Handshake (CLI Getspec)",
+        .prognum   = GLUSTER_HNDSK_PROGRAM,
+        .progver   = GLUSTER_HNDSK_VERSION,
+        .actors    = gluster_cli_getspec_actors,
+        .numactors = GF_HNDSK_MAXVALUE,
+};
+
+
 char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
         [GF_DUMP_NULL] = "NULL",
         [GF_DUMP_DUMP] = "DUMP",
 };
 
 rpc_clnt_prog_t glusterd_dump_prog = {
-        .progname  = "GLUSTERD-DUMP",
-        .prognum   = GLUSTER_DUMP_PROGRAM,
-        .progver   = GLUSTER_DUMP_VERSION,
-        .procnames = glusterd_dump_proc,
+        .progname       = "GLUSTERD-DUMP",
+        .prognum        = GLUSTER_DUMP_PROGRAM,
+        .progver        = GLUSTER_DUMP_VERSION,
+        .procnames      = glusterd_dump_proc,
+};
+
+
+rpcsvc_actor_t glusterd_mgmt_hndsk_actors[] = {
+        [GD_MGMT_HNDSK_NULL]            = {"NULL", GD_MGMT_HNDSK_NULL, NULL,
+                                           NULL, 0},
+        [GD_MGMT_HNDSK_VERSIONS]        = {"MGMT-VERS", GD_MGMT_HNDSK_VERSIONS,
+                                           glusterd_mgmt_hndsk_versions, NULL,
+                                           0},
+        [GD_MGMT_HNDSK_VERSIONS_ACK]    = {"MGMT-VERS-ACK",
+                                           GD_MGMT_HNDSK_VERSIONS_ACK,
+                                           glusterd_mgmt_hndsk_versions_ack,
+                                           NULL, 0},
+};
+
+struct rpcsvc_program glusterd_mgmt_hndsk_prog = {
+        .progname       = "Gluster MGMT Handshake",
+        .prognum        = GD_MGMT_HNDSK_PROGRAM,
+        .progver        = GD_MGMT_HNDSK_VERSION,
+        .actors         = glusterd_mgmt_hndsk_actors,
+        .numactors      = GD_MGMT_HNDSK_MAXVALUE,
+};
+
+char *glusterd_mgmt_hndsk_proc[GD_MGMT_HNDSK_MAXVALUE] = {
+        [GD_MGMT_HNDSK_NULL]          = "NULL",
+        [GD_MGMT_HNDSK_VERSIONS] = "MGMT-VERS",
+        [GD_MGMT_HNDSK_VERSIONS_ACK] = "MGMT-VERS-ACK",
 };
 
+rpc_clnt_prog_t gd_clnt_mgmt_hndsk_prog = {
+        .progname  = "Gluster MGMT Handshake",
+        .prognum   = GD_MGMT_HNDSK_PROGRAM,
+        .progver   = GD_MGMT_HNDSK_VERSION,
+        .procnames = glusterd_mgmt_hndsk_proc,
+};
+
+
 static int
 glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
 {
@@ -279,6 +699,7 @@ glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
         ctx->hostname = gf_strdup (peerinfo->hostname);
         ctx->port = peerinfo->port;
         ctx->req = peerctx->args.req;
+        ctx->dict = peerctx->args.dict;
 
         event->peerinfo = peerinfo;
         event->ctx = ctx;
@@ -296,12 +717,286 @@ out:
         return ret;
 }
 
+
+int
+gd_validate_peer_op_version (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                             dict_t *dict, char **errstr)
+{
+        int              ret  = -1;
+        glusterd_conf_t *conf = NULL;
+        int32_t          peer_op_version = 0;
+        int32_t          peer_min_op_version = 0;
+        int32_t          peer_max_op_version = 0;
+
+        if (!dict && !this && !peerinfo)
+                goto out;
+
+        conf = this->private;
+
+        ret = dict_get_int32 (dict, GD_OP_VERSION_KEY, &peer_op_version);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, GD_MAX_OP_VERSION_KEY,
+                              &peer_max_op_version);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, GD_MIN_OP_VERSION_KEY,
+                              &peer_min_op_version);
+        if (ret)
+                goto out;
+
+        ret = -1;
+        /* Check if peer can support our op_version */
+        if ((peer_max_op_version < conf->op_version) ||
+            (peer_min_op_version > conf->op_version)) {
+                ret = gf_asprintf (errstr, "Peer %s does not support required "
+                                   "op-version", peerinfo->hostname);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name , GF_LOG_DEBUG, "Peer %s %s", peerinfo->hostname,
+                ((ret < 0) ? "rejected" : "accepted"));
+        return ret;
+}
+
+int
+__glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        int                  ret      = -1;
+        int                  op_errno = EINVAL;
+        gf_mgmt_hndsk_rsp    rsp      = {0,};
+        xlator_t            *this     = NULL;
+        call_frame_t        *frame    = NULL;
+        glusterd_peerinfo_t *peerinfo = NULL;
+        glusterd_peerctx_t  *peerctx  = NULL;
+        char                msg[1024] = {0,};
+
+        this = THIS;
+        frame = myframe;
+        peerctx = frame->local;
+        peerinfo = peerctx->peerinfo;
+
+        if (-1 == req->rpc_status) {
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+        if (ret < 0) {
+                snprintf (msg, sizeof (msg), "Failed to decode XDR");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        op_errno = rsp.op_errno;
+        if (-1 == rsp.op_ret) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Failed to get handshake ack from remote server");
+                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        /* TODO: this is hardcoded as of now, but I don't forsee any problems
+         * with this as long as we are properly handshaking operating versions
+         */
+        peerinfo->mgmt = &gd_mgmt_prog;
+        peerinfo->peer = &gd_peer_prog;
+        peerinfo->mgmt_v3 = &gd_mgmt_v3_prog;
+
+        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+
+        if (GD_MODE_ON == peerctx->args.mode) {
+                ret = glusterd_event_connected_inject (peerctx);
+                peerctx->args.req = NULL;
+        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+                peerctx->args.mode = GD_MODE_ON;
+        } else {
+                gf_log (this->name, GF_LOG_WARNING, "unknown mode %d",
+                        peerctx->args.mode);
+        }
+
+        glusterd_friend_sm ();
+
+        ret = 0;
+out:
+
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        if (ret != 0)
+                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+
+        if (rsp.hndsk.hndsk_val)
+                free (rsp.hndsk.hndsk_val);
+
+        return 0;
+}
+
+int
+glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_mgmt_hndsk_version_ack_cbk);
+}
+
+int
+__glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int                  ret       = -1;
+        int                  op_errno  = EINVAL;
+        gf_mgmt_hndsk_rsp    rsp       = {0,};
+        gf_mgmt_hndsk_req    arg       = {{0,}};
+        xlator_t            *this      = NULL;
+        call_frame_t        *frame     = NULL;
+        glusterd_peerinfo_t *peerinfo  = NULL;
+        glusterd_peerctx_t  *peerctx   = NULL;
+        dict_t              *dict      = NULL;
+        dict_t              *rsp_dict  = NULL;
+        glusterd_conf_t     *conf      = NULL;
+        char                 msg[1024] = {0,};
+
+        this = THIS;
+        conf = this->private;
+        frame = myframe;
+        peerctx = frame->local;
+        peerinfo = peerctx->peerinfo;
+
+        if (-1 == req->rpc_status) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+        if (ret < 0) {
+                snprintf (msg, sizeof (msg), "Failed to decode management "
+                          "handshake response");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, dict, rsp.hndsk.hndsk_val,
+                                      rsp.hndsk.hndsk_len, ret, op_errno,
+                                      out);
+
+        op_errno = rsp.op_errno;
+        if (-1 == rsp.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get the 'versions' from peer (%s)",
+                        req->conn->trans->peerinfo.identifier);
+                goto out;
+        }
+
+        /* Check if peer can be part of cluster */
+        ret = gd_validate_peer_op_version (this, peerinfo, dict,
+                                           &peerctx->errstr);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to validate the operating version of peer (%s)",
+                        peerinfo->hostname);
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict)
+                goto out;
+
+        ret = dict_set_int32 (rsp_dict, GD_OP_VERSION_KEY, conf->op_version);
+        if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to set operating version in dict");
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, rsp_dict, (&arg.hndsk.hndsk_val),
+                                    arg.hndsk.hndsk_len, op_errno, out);
+
+        ret = glusterd_submit_request (peerctx->peerinfo->rpc, &arg, frame,
+                                       &gd_clnt_mgmt_hndsk_prog,
+                                       GD_MGMT_HNDSK_VERSIONS_ACK, NULL, this,
+                                       glusterd_mgmt_hndsk_version_ack_cbk,
+                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+
+out:
+        if (ret) {
+                frame->local = NULL;
+                STACK_DESTROY (frame->root);
+                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+        }
+
+        if (rsp.hndsk.hndsk_val)
+                free (rsp.hndsk.hndsk_val);
+
+        if (arg.hndsk.hndsk_val)
+                GF_FREE (arg.hndsk.hndsk_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        return 0;
+}
+
+int
+glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_mgmt_hndsk_version_cbk);
+}
+
+int
+glusterd_mgmt_handshake (xlator_t *this, glusterd_peerctx_t *peerctx)
+{
+        call_frame_t        *frame    = NULL;
+        gf_mgmt_hndsk_req    req      = {{0,},};
+        int                  ret      = -1;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->local = peerctx;
+
+        ret = glusterd_submit_request (peerctx->peerinfo->rpc, &req, frame,
+                                       &gd_clnt_mgmt_hndsk_prog,
+                                       GD_MGMT_HNDSK_VERSIONS, NULL, this,
+                                       glusterd_mgmt_hndsk_version_cbk,
+                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        ret = 0;
+out:
+        if (ret && frame)
+                STACK_DESTROY (frame->root);
+
+        return ret;
+}
+
 int
 glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
                                 gf_prog_detail *prog)
 {
-        gf_prog_detail *trav     = NULL;
-        int             ret      = -1;
+        gf_prog_detail  *trav   = NULL;
+        int             ret     = -1;
 
         if (!peerinfo || !prog)
                 goto out;
@@ -309,45 +1004,80 @@ glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
         trav = prog;
 
         while (trav) {
-                /* Select 'programs' */
+                ret = -1;
                 if ((gd_mgmt_prog.prognum == trav->prognum) &&
                     (gd_mgmt_prog.progver == trav->progver)) {
                         peerinfo->mgmt = &gd_mgmt_prog;
                         ret = 0;
                 }
+
                 if ((gd_peer_prog.prognum == trav->prognum) &&
                     (gd_peer_prog.progver == trav->progver)) {
                         peerinfo->peer = &gd_peer_prog;
                         ret = 0;
                 }
+
                 if (ret) {
                         gf_log ("", GF_LOG_DEBUG,
                                 "%s (%"PRId64":%"PRId64") not supported",
                                 trav->progname, trav->prognum,
                                 trav->progver);
                 }
+
                 trav = trav->next;
         }
 
         if (peerinfo->mgmt) {
-                gf_log ("", GF_LOG_INFO,
-                        "Using Program %s, Num (%d), Version (%d)",
-                        peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
-                        peerinfo->mgmt->progver);
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
+                         peerinfo->mgmt->progver);
         }
+
         if (peerinfo->peer) {
-                gf_log ("", GF_LOG_INFO,
-                        "Using Program %s, Num (%d), Version (%d)",
-                        peerinfo->peer->progname, peerinfo->peer->prognum,
-                        peerinfo->peer->progver);
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->peer->progname, peerinfo->peer->prognum,
+                         peerinfo->peer->progver);
+        }
+
+        if (peerinfo->mgmt_v3) {
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->mgmt_v3->progname,
+                         peerinfo->mgmt_v3->prognum,
+                         peerinfo->mgmt_v3->progver);
         }
 
+        ret = 0;
+out:
+        return ret;
+
+}
+
+static gf_boolean_t
+_mgmt_hndsk_prog_present (gf_prog_detail *prog) {
+        gf_boolean_t    ret = _gf_false;
+        gf_prog_detail  *trav = NULL;
+
+        GF_ASSERT (prog);
+
+        trav = prog;
+
+        while (trav) {
+                if ((trav->prognum == GD_MGMT_HNDSK_PROGRAM) &&
+                    (trav->progver == GD_MGMT_HNDSK_VERSION)) {
+                        ret = _gf_true;
+                        goto out;
+                }
+                trav = trav->next;
+        }
 out:
         return ret;
 }
 
 int
-glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
                                 int count, void *myframe)
 {
         int                  ret      = -1;
@@ -358,26 +1088,51 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
         call_frame_t        *frame    = NULL;
         glusterd_peerinfo_t *peerinfo = NULL;
         glusterd_peerctx_t  *peerctx  = NULL;
+        glusterd_conf_t     *conf     = NULL;
+        char                msg[1024] = {0,};
 
         this = THIS;
+        conf = this->private;
         frame = myframe;
         peerctx = frame->local;
         peerinfo = peerctx->peerinfo;
 
         if (-1 == req->rpc_status) {
-                gf_log ("", GF_LOG_ERROR,
-                        "error through RPC layer, retry again later");
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "failed to decode XDR");
+                snprintf (msg, sizeof (msg), "Failed to decode XDR");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
                 goto out;
         }
         if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "failed to get the 'versions' from remote server");
+                snprintf (msg, sizeof (msg),
+                          "Failed to get the 'versions' from remote server");
+                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        if (_mgmt_hndsk_prog_present (rsp.prog)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Proceeding to op-version handshake with peer %s",
+                        peerinfo->hostname);
+                ret = glusterd_mgmt_handshake (this, peerctx);
+                goto out;
+        } else if (conf->op_version > 1) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Peer %s does not support required op-version",
+                          peerinfo->hostname);
+                peerctx->errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
@@ -400,10 +1155,11 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
                         peerctx->args.mode);
         }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        glusterd_friend_sm();
+        glusterd_op_sm();
 
         ret = 0;
+
 out:
 
         /* don't use GF_FREE, buffer was allocated by libc */
@@ -428,8 +1184,16 @@ out:
 
 
 int
-glusterd_peer_handshake (xlator_t *this, struct rpc_clnt *rpc,
-                         glusterd_peerctx_t *peerctx)
+glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_peer_dump_version_cbk);
+}
+
+int
+glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
+                            glusterd_peerctx_t *peerctx)
 {
         call_frame_t        *frame    = NULL;
         gf_dump_req          req      = {0,};
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.c b/xlators/mgmt/glusterd/src/glusterd-hooks.c
new file mode 100644
index 000000000..352b6ba11
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.c
@@ -0,0 +1,559 @@
+/*
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "dict.h"
+#include "xlator.h"
+#include "logging.h"
+#include "run.h"
+#include "defaults.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-hooks.h"
+
+#include <fnmatch.h>
+
+#define EMPTY ""
+char glusterd_hook_dirnames[GD_OP_MAX][256] =
+{
+        [GD_OP_NONE]                    = EMPTY,
+        [GD_OP_CREATE_VOLUME]           = "create",
+        [GD_OP_START_BRICK]             = EMPTY,
+        [GD_OP_STOP_BRICK]              = EMPTY,
+        [GD_OP_DELETE_VOLUME]           = "delete",
+        [GD_OP_START_VOLUME]            = "start",
+        [GD_OP_STOP_VOLUME]             = "stop",
+        [GD_OP_DEFRAG_VOLUME]           = EMPTY,
+        [GD_OP_ADD_BRICK]               = "add-brick",
+        [GD_OP_REMOVE_BRICK]            = "remove-brick",
+        [GD_OP_REPLACE_BRICK]           = EMPTY,
+        [GD_OP_SET_VOLUME]              = "set",
+        [GD_OP_RESET_VOLUME]            = EMPTY,
+        [GD_OP_SYNC_VOLUME]             = EMPTY,
+        [GD_OP_LOG_ROTATE]              = EMPTY,
+        [GD_OP_GSYNC_CREATE]            = "gsync-create",
+        [GD_OP_GSYNC_SET]               = EMPTY,
+        [GD_OP_PROFILE_VOLUME]          = EMPTY,
+        [GD_OP_QUOTA]                   = EMPTY,
+        [GD_OP_STATUS_VOLUME]           = EMPTY,
+        [GD_OP_REBALANCE]               = EMPTY,
+        [GD_OP_HEAL_VOLUME]             = EMPTY,
+        [GD_OP_STATEDUMP_VOLUME]        = EMPTY,
+        [GD_OP_LIST_VOLUME]             = EMPTY,
+        [GD_OP_CLEARLOCKS_VOLUME]       = EMPTY,
+        [GD_OP_DEFRAG_BRICK_VOLUME]     = EMPTY,
+};
+#undef EMPTY
+
+static inline gf_boolean_t
+glusterd_is_hook_enabled (char *script)
+{
+        return (script[0] == 'S');
+}
+
+int
+glusterd_hooks_create_hooks_directory (char *basedir)
+{
+        int  ret                                        = -1;
+        int  op                                         = GD_OP_NONE;
+        int  type                                       = GD_COMMIT_HOOK_NONE;
+        char version_dir[PATH_MAX]                      = {0, };
+        char path[PATH_MAX]                             = {0, };
+        char *cmd_subdir                                = NULL;
+        char type_subdir[GD_COMMIT_HOOK_MAX][256]       = {{0, },
+                                                           "pre",
+                                                           "post"};
+        glusterd_conf_t *priv                           = NULL;
+
+        priv = THIS->private;
+
+        snprintf (path, sizeof (path), "%s/hooks", basedir);
+        ret = mkdir_p (path, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to create %s due"
+                         "to %s", path, strerror (errno));
+                goto out;
+        }
+
+        GLUSTERD_GET_HOOKS_DIR (version_dir, GLUSTERD_HOOK_VER, priv);
+        ret = mkdir_p (version_dir, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to create %s due "
+                        "to %s", version_dir, strerror (errno));
+                goto out;
+        }
+
+        for (op = GD_OP_NONE+1; op < GD_OP_MAX; op++) {
+                cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
+                if (strlen (cmd_subdir) == 0)
+                        continue;
+
+                snprintf (path, sizeof (path), "%s/%s", version_dir,
+                          cmd_subdir);
+                ret = mkdir_p (path, 0777, _gf_true);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_CRITICAL,
+                                "Unable to create %s due to %s",
+                                path, strerror (errno));
+                        goto out;
+                }
+
+                for (type = GD_COMMIT_HOOK_PRE; type < GD_COMMIT_HOOK_MAX;
+                     type++) {
+                        snprintf (path, sizeof (path), "%s/%s/%s",
+                                  version_dir, cmd_subdir, type_subdir[type]);
+                        ret = mkdir_p (path, 0777, _gf_true);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_CRITICAL,
+                                        "Unable to create %s due to %s",
+                                        path, strerror (errno));
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+char*
+glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op)
+{
+        GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
+
+        return glusterd_hook_dirnames[op];
+}
+
+void
+glusterd_hooks_add_working_dir (runner_t *runner, glusterd_conf_t *priv)
+{
+        runner_argprintf (runner, "--gd-workdir=%s", priv->workdir);
+}
+
+void
+glusterd_hooks_add_op (runner_t *runner, char *op)
+{
+        runner_argprintf (runner, "--volume-op=%s", op);
+}
+
+void
+glusterd_hooks_add_hooks_version (runner_t* runner)
+{
+        runner_argprintf (runner, "--version=%d", GLUSTERD_HOOK_VER);
+}
+
+int
+glusterd_hooks_set_volume_args (dict_t *dict, runner_t *runner)
+{
+        int     i               = 0;
+        int     count           = 0;
+        int     ret             = -1;
+        char    query[1024]     = {0,};
+        char    *key            = NULL;
+        char    *value          = NULL;
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+
+        /* This will not happen unless op_ctx
+         * is corrupted*/
+        if (!count)
+                goto out;
+
+        runner_add_arg (runner, "-o");
+        for (i = 1; (ret == 0); i++) {
+                snprintf (query, sizeof (query), "key%d", i);
+                ret = dict_get_str (dict, query, &key);
+                if (ret)
+                       continue;
+
+                snprintf (query, sizeof (query), "value%d", i);
+                ret = dict_get_str (dict, query, &value);
+                if (ret)
+                       continue;
+
+                runner_argprintf (runner, "%s=%s", key, value);
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+glusterd_hooks_add_op_args (runner_t *runner, glusterd_op_t op,
+                            dict_t *op_ctx, glusterd_commit_hook_type_t type)
+{
+        char                   *hooks_args       = NULL;
+        int                     vol_count        = 0;
+        gf_boolean_t            truth            = _gf_false;
+        glusterd_volinfo_t      *voliter         = NULL;
+        glusterd_conf_t         *priv            = NULL;
+        int                     ret              = -1;
+
+        priv = THIS->private;
+        list_for_each_entry (voliter, &priv->volumes,
+                             vol_list) {
+                if (glusterd_is_volume_started (voliter))
+                        vol_count++;
+        }
+
+        ret = 0;
+        switch (op) {
+                case GD_OP_START_VOLUME:
+                        if (type == GD_COMMIT_HOOK_PRE &&
+                            vol_count == 0)
+                                truth = _gf_true;
+
+                        else if (type == GD_COMMIT_HOOK_POST &&
+                                 vol_count == 1)
+                                truth = _gf_true;
+
+                        else
+                                truth = _gf_false;
+
+                        runner_argprintf (runner, "--first=%s",
+                                          truth? "yes":"no");
+
+                        glusterd_hooks_add_hooks_version (runner);
+                        glusterd_hooks_add_op (runner, "start");
+                        glusterd_hooks_add_working_dir (runner, priv);
+
+                        break;
+
+                case GD_OP_STOP_VOLUME:
+                        if (type == GD_COMMIT_HOOK_PRE &&
+                            vol_count == 1)
+                                truth = _gf_true;
+
+                        else if (type == GD_COMMIT_HOOK_POST &&
+                                 vol_count == 0)
+                                truth = _gf_true;
+
+                        else
+                                truth = _gf_false;
+
+                        runner_argprintf (runner, "--last=%s",
+                                          truth? "yes":"no");
+                        break;
+
+                case GD_OP_SET_VOLUME:
+                        ret = glusterd_hooks_set_volume_args (op_ctx, runner);
+                        break;
+
+                case GD_OP_GSYNC_CREATE:
+                        ret = dict_get_str (op_ctx, "hooks_args", &hooks_args);
+                        if (ret)
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "No Hooks Arguments.");
+                        else
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Hooks Args = %s", hooks_args);
+                        if (hooks_args)
+                                runner_argprintf (runner, "%s", hooks_args);
+                        break;
+
+                case GD_OP_ADD_BRICK:
+                        glusterd_hooks_add_hooks_version (runner);
+                        glusterd_hooks_add_op (runner, "add-brick");
+                        glusterd_hooks_add_working_dir (runner, priv);
+
+                default:
+                        break;
+
+        }
+
+        return ret;
+}
+
+int
+glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                          glusterd_commit_hook_type_t type)
+{
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        runner_t        runner                  = {0, };
+        struct dirent   *entry                  = NULL;
+        DIR             *hookdir                = NULL;
+        char            *volname                = NULL;
+        char            **lines                 = NULL;
+        int             N                       = 8; /*arbitrary*/
+        int             lineno                  = 0;
+        int             line_count              = 0;
+        int             ret                     = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        ret = dict_get_str (op_ctx, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to get volname "
+                        "from operation context");
+                goto out;
+        }
+
+        hookdir = opendir (hooks_path);
+        if (!hookdir) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s, due "
+                        "to %s", hooks_path, strerror (errno));
+                goto out;
+        }
+
+        lines = GF_CALLOC (1, N * sizeof (*lines), gf_gld_mt_charptr);
+        if (!lines) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = -1;
+        line_count = 0;
+        glusterd_for_each_entry (entry, hookdir);
+        while (entry) {
+                if (line_count == N-1) {
+                        N *= 2;
+                        lines = GF_REALLOC (lines, N * sizeof (char *));
+                        if (!lines)
+                                goto out;
+                }
+
+                if (glusterd_is_hook_enabled (entry->d_name)) {
+                        lines[line_count] = gf_strdup (entry->d_name);
+                        line_count++;
+                }
+
+                glusterd_for_each_entry (entry, hookdir);
+        }
+
+        lines[line_count] = NULL;
+        lines = GF_REALLOC (lines, (line_count + 1) * sizeof (char *));
+        if (!lines)
+                goto out;
+
+        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
+
+        for (lineno = 0; lineno < line_count; lineno++) {
+
+                runinit (&runner);
+                runner_argprintf (&runner, "%s/%s", hooks_path, lines[lineno]);
+                /*Add future command line arguments to hook scripts below*/
+                runner_argprintf (&runner, "--volname=%s", volname);
+                ret = glusterd_hooks_add_op_args (&runner, op, op_ctx, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to add "
+                                "command specific arguments");
+                        goto out;
+                }
+
+                ret = runner_run_reuse (&runner);
+                if (ret) {
+                        runner_log (&runner, this->name, GF_LOG_ERROR,
+                                    "Failed to execute script");
+                } else {
+                        runner_log (&runner, this->name, GF_LOG_INFO,
+                                    "Ran script");
+                }
+                runner_end (&runner);
+        }
+
+        ret = 0;
+out:
+        if (lines) {
+                for (lineno = 0; lineno < line_count+1; lineno++)
+                        GF_FREE (lines[lineno]);
+
+                GF_FREE (lines);
+        }
+
+        if (hookdir)
+                closedir (hookdir);
+
+        return ret;
+}
+
+int
+glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
+                                  dict_t *op_ctx)
+{
+        int                     ret     = -1;
+        glusterd_hooks_stub_t   *stub   = NULL;
+        glusterd_hooks_private_t *hooks_priv = NULL;
+        glusterd_conf_t         *conf   = NULL;
+
+        conf = THIS->private;
+        hooks_priv = conf->hooks_priv;
+
+        ret = glusterd_hooks_stub_init (&stub, scriptdir, op, op_ctx);
+        if (ret)
+                goto out;
+
+        pthread_mutex_lock (&hooks_priv->mutex);
+        {
+                hooks_priv->waitcount++;
+                list_add_tail (&stub->all_hooks, &hooks_priv->list);
+                pthread_cond_signal (&hooks_priv->cond);
+        }
+        pthread_mutex_unlock (&hooks_priv->mutex);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
+                          glusterd_op_t op, dict_t *op_ctx)
+{
+        int                     ret             = -1;
+        glusterd_hooks_stub_t   *hooks_stub     = NULL;
+
+        GF_ASSERT (stub);
+        if (!stub)
+                goto out;
+
+        hooks_stub = GF_CALLOC (1, sizeof (*hooks_stub),
+                                gf_gld_mt_hooks_stub_t);
+        if (!hooks_stub)
+                goto out;
+
+        INIT_LIST_HEAD (&hooks_stub->all_hooks);
+        hooks_stub->op = op;
+        hooks_stub->scriptdir = gf_strdup (scriptdir);
+        if (!hooks_stub->scriptdir)
+                goto out;
+
+        hooks_stub->op_ctx = dict_copy_with_ref (op_ctx, hooks_stub->op_ctx);
+        if (!hooks_stub->op_ctx)
+                goto out;
+
+        *stub = hooks_stub;
+        ret = 0;
+out:
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to initialize "
+                        "post hooks stub");
+                glusterd_hooks_stub_cleanup (hooks_stub);
+        }
+
+        return ret;
+}
+
+void
+glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub)
+{
+        if (!stub) {
+                gf_log_callingfn (THIS->name, GF_LOG_WARNING,
+                                  "hooks_stub is NULL");
+                return;
+        }
+
+        if (stub->op_ctx)
+                dict_unref (stub->op_ctx);
+
+        GF_FREE (stub->scriptdir);
+
+        GF_FREE (stub);
+}
+
+static void*
+hooks_worker (void *args)
+{
+        glusterd_conf_t *conf = NULL;
+        glusterd_hooks_private_t *hooks_priv = NULL;
+        glusterd_hooks_stub_t *stub = NULL;
+
+        THIS = args;
+        conf = THIS->private;
+        hooks_priv = conf->hooks_priv;
+
+        for (;;) {
+                pthread_mutex_lock (&hooks_priv->mutex);
+                {
+                        while (list_empty (&hooks_priv->list)) {
+                                pthread_cond_wait (&hooks_priv->cond,
+                                                   &hooks_priv->mutex);
+                        }
+                        stub = list_entry (hooks_priv->list.next,
+                                           glusterd_hooks_stub_t,
+                                           all_hooks);
+                        list_del_init (&stub->all_hooks);
+                        hooks_priv->waitcount--;
+
+                }
+                pthread_mutex_unlock (&hooks_priv->mutex);
+
+                glusterd_hooks_run_hooks (stub->scriptdir, stub->op,
+                                          stub->op_ctx, GD_COMMIT_HOOK_POST);
+                glusterd_hooks_stub_cleanup (stub);
+        }
+
+        return NULL;
+}
+
+int
+glusterd_hooks_priv_init (glusterd_hooks_private_t **new)
+{
+        int                      ret            = -1;
+        glusterd_hooks_private_t *hooks_priv    = NULL;
+
+        if (!new)
+                goto out;
+
+        hooks_priv = GF_CALLOC (1, sizeof (*hooks_priv),
+                                gf_gld_mt_hooks_priv_t);
+        if (!hooks_priv)
+                goto out;
+
+        pthread_mutex_init (&hooks_priv->mutex, NULL);
+        pthread_cond_init (&hooks_priv->cond, NULL);
+        INIT_LIST_HEAD (&hooks_priv->list);
+        hooks_priv->waitcount = 0;
+
+        *new = hooks_priv;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_hooks_spawn_worker (xlator_t *this)
+{
+        int                       ret          = -1;
+        glusterd_conf_t           *conf        = NULL;
+        glusterd_hooks_private_t  *hooks_priv  = NULL;
+
+
+        ret = glusterd_hooks_priv_init (&hooks_priv);
+        if (ret)
+                goto out;
+
+        conf = this->private;
+        conf->hooks_priv = hooks_priv;
+        ret = pthread_create (&hooks_priv->worker, NULL, hooks_worker,
+                              (void *)this);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn post "
+                        "hooks worker thread");
+out:
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.h b/xlators/mgmt/glusterd/src/glusterd-hooks.h
new file mode 100644
index 000000000..c597ddd2a
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.h
@@ -0,0 +1,89 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_HOOKS_H_
+#define _GLUSTERD_HOOKS_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <fnmatch.h>
+
+#define GLUSTERD_GET_HOOKS_DIR(path, version, priv) \
+        snprintf (path, PATH_MAX, "%s/hooks/%d", priv->workdir,\
+                  version);
+
+#define GLUSTERD_HOOK_VER       1
+
+#define GD_HOOKS_SPECIFIC_KEY   "user.*"
+
+typedef enum glusterd_commit_hook_type {
+        GD_COMMIT_HOOK_NONE = 0,
+        GD_COMMIT_HOOK_PRE,
+        GD_COMMIT_HOOK_POST,
+        GD_COMMIT_HOOK_MAX
+} glusterd_commit_hook_type_t;
+
+typedef struct hooks_private {
+        struct list_head        list;
+        int                     waitcount; //debug purposes
+        pthread_mutex_t         mutex;
+        pthread_cond_t          cond;
+        pthread_t               worker;
+} glusterd_hooks_private_t;
+
+typedef struct hooks_stub {
+        struct list_head        all_hooks;
+        char                    *scriptdir;
+        glusterd_op_t           op;
+        dict_t                  *op_ctx;
+
+} glusterd_hooks_stub_t;
+
+
+static inline gf_boolean_t
+is_key_glusterd_hooks_friendly (char *key)
+{
+        gf_boolean_t is_friendly = _gf_false;
+
+        /* This is very specific to hooks friendly behavior */
+        if (fnmatch (GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "user namespace key %s", key);
+                is_friendly = _gf_true;
+        }
+
+        return is_friendly;
+}
+
+int
+glusterd_hooks_create_hooks_directory (char *basedir);
+
+char *
+glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op);
+
+int
+glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                          glusterd_commit_hook_type_t type);
+int
+glusterd_hooks_spawn_worker (xlator_t *this);
+
+int
+glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
+                          glusterd_op_t op, dict_t *op_ctx);
+void
+glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub);
+
+int
+glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
+                                      dict_t *op_ctx);
+int
+glusterd_hooks_priv_init (glusterd_hooks_private_t **new);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c
new file mode 100644
index 000000000..9e8bbc21b
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.c
@@ -0,0 +1,213 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "glusterd-locks.h"
+#include "run.h"
+#include "syscall.h"
+
+#include <signal.h>
+
+/* Initialize the global vol-lock list(dict) when
+ * glusterd is spawned */
+int32_t
+glusterd_vol_lock_init ()
+{
+        int32_t             ret = -1;
+        xlator_t           *this   = NULL;
+        glusterd_conf_t    *priv   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        priv->vol_lock = dict_new ();
+        if (!priv->vol_lock)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* Destroy the global vol-lock list(dict) when
+ * glusterd cleanup is performed */
+void
+glusterd_vol_lock_fini ()
+{
+        xlator_t           *this   = NULL;
+        glusterd_conf_t    *priv   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (priv->vol_lock)
+                dict_unref (priv->vol_lock);
+}
+
+int32_t
+glusterd_get_vol_lock_owner (char *volname, uuid_t *uuid)
+{
+        int32_t                 ret      = -1;
+        glusterd_vol_lock_obj  *lock_obj = NULL;
+        glusterd_conf_t        *priv     = NULL;
+        uuid_t                  no_owner = {0,};
+        xlator_t               *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!volname || !uuid) {
+                gf_log ("", GF_LOG_ERROR, "volname or uuid is null.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin (priv->vol_lock, volname, (void **) &lock_obj);
+        if (!ret)
+                uuid_copy (*uuid, lock_obj->lock_owner);
+        else
+                uuid_copy (*uuid, no_owner);
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_volume_lock (char *volname, uuid_t uuid)
+{
+        int32_t                 ret      = -1;
+        glusterd_vol_lock_obj  *lock_obj = NULL;
+        glusterd_conf_t        *priv     = NULL;
+        uuid_t                  owner    = {0};
+        xlator_t               *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!volname) {
+                gf_log ("", GF_LOG_ERROR, "volname is null.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_vol_lock_owner (volname, &owner);
+        if (ret) {
+                gf_log ("", GF_LOG_WARNING,
+                        "Unable to get volume lock owner");
+                goto out;
+        }
+
+        /* If the lock has already been held for the given volume
+         * we fail */
+        if (!uuid_is_null (owner)) {
+                gf_log ("", GF_LOG_WARNING, "Unable to acquire lock. "
+                        "Lock for %s held by %s", volname,
+                        uuid_utoa (owner));
+                ret = -1;
+                goto out;
+        }
+
+        lock_obj = GF_CALLOC (1, sizeof(glusterd_vol_lock_obj),
+                              gf_common_mt_vol_lock_obj_t);
+        if (!lock_obj) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (lock_obj->lock_owner, uuid);
+
+        ret = dict_set_bin (priv->vol_lock, volname, lock_obj,
+                            sizeof(glusterd_vol_lock_obj));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to set lock owner "
+                                          "in volume lock");
+                if (lock_obj)
+                        GF_FREE (lock_obj);
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Lock for %s successfully held by %s",
+                volname, uuid_utoa (uuid));
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_volume_unlock (char *volname, uuid_t uuid)
+{
+        int32_t                 ret      = -1;
+        glusterd_conf_t        *priv     = NULL;
+        uuid_t                  owner    = {0};
+        xlator_t               *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!volname) {
+                gf_log ("", GF_LOG_ERROR, "volname is null.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_vol_lock_owner (volname, &owner);
+        if (ret)
+                goto out;
+
+        if (uuid_is_null (owner)) {
+                gf_log ("", GF_LOG_WARNING, "Lock for %s not held", volname);
+                ret = -1;
+                goto out;
+        }
+
+        ret = uuid_compare (uuid, owner);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, "Lock owner mismatch. "
+                        "Lock for %s held by %s",
+                        volname, uuid_utoa (owner));
+                goto out;
+        }
+
+        /* Removing the volume lock from the global list */
+        dict_del (priv->vol_lock, volname);
+
+        gf_log ("", GF_LOG_DEBUG, "Lock for %s successfully released",
+                volname);
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.h b/xlators/mgmt/glusterd/src/glusterd-locks.h
new file mode 100644
index 000000000..6e3f56f9c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.h
@@ -0,0 +1,37 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_LOCKS_H_
+#define _GLUSTERD_LOCKS_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+typedef struct glusterd_volume_lock_object_ {
+        uuid_t              lock_owner;
+} glusterd_vol_lock_obj;
+
+int32_t
+glusterd_vol_lock_init ();
+
+void
+glusterd_vol_lock_fini ();
+
+int32_t
+glusterd_get_vol_lock_owner (char *volname, uuid_t *uuid);
+
+int32_t
+glusterd_volume_lock (char *volname, uuid_t uuid);
+
+int32_t
+glusterd_volume_unlock (char *volname, uuid_t uuid);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-log-ops.c b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
index 93e037f87..33bd95c03 100644
--- a/xlators/mgmt/glusterd/src/glusterd-log-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -34,18 +24,22 @@
 #include <signal.h>
 
 int
-glusterd_handle_log_rotate (rpcsvc_request_t *req)
+__glusterd_handle_log_rotate (rpcsvc_request_t *req)
 {
         int32_t                 ret     = -1;
         gf_cli_req              cli_req = {{0,}};
         dict_t                 *dict    = NULL;
         glusterd_op_t           cli_op = GD_OP_LOG_ROTATE;
         char                   *volname = NULL;
+        char                   msg[2048] = {0,};
+        xlator_t               *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -59,44 +53,50 @@ glusterd_handle_log_rotate (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (msg, sizeof (msg), "Failed to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received log rotate req "
+        gf_log (this->name, GF_LOG_INFO, "Received log rotate req "
                 "for volume %s", volname);
 
         ret = dict_set_uint64 (dict, "rotate-key", (uint64_t)time (NULL));
         if (ret)
                 goto out;
 
-        ret = glusterd_op_begin (req, GD_OP_LOG_ROTATE, dict);
+        ret = glusterd_op_begin_synctask (req, GD_OP_LOG_ROTATE, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        if (ret)
+        if (ret) {
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, msg);
+        }
 
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
+        free (cli_req.dict.dict_val);
         return ret;
 }
 
+int
+glusterd_handle_log_rotate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_log_rotate);
+}
+
 /* op-sm */
 int
 glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr)
@@ -142,16 +142,13 @@ glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        if (strchr (brick, ':')) {
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo, NULL,
-                                                              GF_PATH_COMPLETE);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Incorrect brick %s "
-                                  "for volume %s", brick, volname);
-                        gf_log ("", GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo, NULL);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Incorrect brick %s "
+                          "for volume %s", brick, volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
         }
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
@@ -170,7 +167,6 @@ glusterd_op_log_rotate (dict_t *dict)
         xlator_t             *this               = NULL;
         char                 *volname            = NULL;
         char                 *brick              = NULL;
-        char                  path[PATH_MAX]     = {0,};
         char                  logfile[PATH_MAX]  = {0,};
         char                  pidfile[PATH_MAX]  = {0,};
         FILE                 *file               = NULL;
@@ -202,15 +198,11 @@ glusterd_op_log_rotate (dict_t *dict)
         if (ret)
                 goto cont;
 
-        if (!strchr (brick, ':'))
-                brick = NULL;
-        else {
-                ret = glusterd_brickinfo_from_brick (brick, &tmpbrkinfo);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "cannot get brickinfo from brick");
-                        goto out;
-                }
+        ret = glusterd_brickinfo_new_from_brick (brick, &tmpbrkinfo);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "cannot get brickinfo from brick");
+                goto out;
         }
 
 cont:
@@ -220,7 +212,7 @@ cont:
 
         ret = -1;
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (uuid_compare (brickinfo->uuid, priv->uuid))
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
                         continue;
 
                 if (brick &&
@@ -230,10 +222,7 @@ cont:
 
                 valid_brick = 1;
 
-                GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
-                                            brickinfo->path);
-
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
                 file = fopen (pidfile, "r+");
                 if (!file) {
                         gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index 3bb446d1a..51057c274 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __GLUSTERD_MEM_TYPES_H__
 #define __GLUSTERD_MEM_TYPES_H__
 
@@ -70,9 +60,14 @@ typedef enum gf_gld_mem_types_ {
         gf_gld_mt_mount_comp_container          = gf_common_mt_end + 44,
         gf_gld_mt_mount_component               = gf_common_mt_end + 45,
         gf_gld_mt_mount_spec                    = gf_common_mt_end + 46,
-        gf_gld_mt_nodesrv_t                     = gf_common_mt_end + 47,
-        gf_gld_mt_charptr                       = gf_common_mt_end + 48,
-        gf_gld_mt_end                           = gf_common_mt_end + 49,
+        gf_gld_mt_georep_meet_spec              = gf_common_mt_end + 47,
+        gf_gld_mt_nodesrv_t                     = gf_common_mt_end + 48,
+        gf_gld_mt_charptr                       = gf_common_mt_end + 49,
+        gf_gld_mt_hooks_stub_t                  = gf_common_mt_end + 50,
+        gf_gld_mt_hooks_priv_t                  = gf_common_mt_end + 51,
+        gf_gld_mt_mop_commit_req_t              = gf_common_mt_end + 52,
+        gf_gld_mt_int                           = gf_common_mt_end + 53,
+        gf_gld_mt_end                           = gf_common_mt_end + 54,
 } gf_gld_mem_types_t;
 #endif
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
index 4bd062c14..4ce441da8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -241,15 +231,17 @@ parse_mount_pattern_desc (gf_mount_spec_t *mspec, char *pdesc)
 
 const char *georep_mnt_desc_template =
         "SUP("
-                "xlator-option=\\*-dht.assert-no-child-down=true "
                 "volfile-server=localhost "
                 "client-pid=%d "
-                "volfile-id=%s "
                 "user-map-root=%s "
         ")"
         "SUB+("
                 "log-file="DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"*/* "
                 "log-level=* "
+                "volfile-id=* "
+        ")"
+        "MEET("
+                "%s"
         ")";
 
 const char *hadoop_mnt_desc_template =
@@ -265,18 +257,69 @@ const char *hadoop_mnt_desc_template =
         ")";
 
 int
-make_georep_mountspec (gf_mount_spec_t *mspec, const char *volname,
+make_georep_mountspec (gf_mount_spec_t *mspec, const char *volnames,
                        char *user)
 {
         char *georep_mnt_desc = NULL;
+        char *meetspec        = NULL;
+        char *vols            = NULL;
+        char *vol             = NULL;
+        char *p               = NULL;
+        char *savetok         = NULL;
+        char *fa[3]           = {0,};
+        size_t siz            = 0;
+        int vc                = 0;
+        int i                 = 0;
         int ret               = 0;
 
+        vols = gf_strdup ((char *)volnames);
+        if (!vols)
+                goto out;
+
+        for (vc = 1, p = vols; *p; p++) {
+                if (*p == ',')
+                        vc++;
+        }
+        siz = strlen (volnames) + vc * strlen("volfile-id=");
+        meetspec = GF_CALLOC (1, siz + 1, gf_gld_mt_georep_meet_spec);
+        if (!meetspec)
+                goto out;
+
+        for (p = vols;;) {
+                vol = strtok_r (p, ",", &savetok);
+                if (!vol) {
+                        GF_ASSERT (vc == 0);
+                        break;
+                }
+                p = NULL;
+                strcat (meetspec, "volfile-id=");
+                strcat (meetspec, vol);
+                if (--vc > 0)
+                        strcat (meetspec, " ");
+        }
+
         ret = gf_asprintf (&georep_mnt_desc, georep_mnt_desc_template,
-                           GF_CLIENT_PID_GSYNCD, volname, user);
-        if (ret == -1)
-                return ret;
+                           GF_CLIENT_PID_GSYNCD, user, meetspec);
+        if (ret == -1) {
+                georep_mnt_desc = NULL;
+                goto out;
+        }
+
+        ret = parse_mount_pattern_desc (mspec, georep_mnt_desc);
 
-        return parse_mount_pattern_desc (mspec, georep_mnt_desc);
+ out:
+        fa[0] = meetspec;
+        fa[1] = vols;
+        fa[2] = georep_mnt_desc;
+
+        for (i = 0; i < 3; i++) {
+                if (fa[i] == NULL)
+                        ret = -1;
+                else
+                        GF_FREE (fa[i]);
+        }
+
+        return ret;
 }
 
 int
@@ -643,8 +686,7 @@ glusterd_do_mount (char *label, dict_t *argdict, char **path, int *op_errno)
                 *path = cookie;
         }
 
-        if (mtptemp)
-                GF_FREE (mtptemp);
+        GF_FREE (mtptemp);
 
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
index 729cf3fbc..426252ebe 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #define MB_HIVE "mb_hive"
 
 typedef enum {
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index bda5e61e4..e3ae369e4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -45,7 +35,9 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
+#include "glusterd-hooks.h"
 #include "glusterd-volgen.h"
+#include "glusterd-locks.h"
 #include "syscall.h"
 #include "cli1-xdr.h"
 #include "common-utils.h"
@@ -55,9 +47,274 @@
 #include <signal.h>
 #include <sys/wait.h>
 
+#define ALL_VOLUME_OPTION_CHECK(volname, key, ret, op_errstr, label)           \
+        do {                                                                   \
+                gf_boolean_t    _all = !strcmp ("all", volname);               \
+                gf_boolean_t    _ratio = !strcmp (key,                         \
+                                                  GLUSTERD_QUORUM_RATIO_KEY);  \
+                if (_all && !_ratio) {                                         \
+                        ret = -1;                                              \
+                        *op_errstr = gf_strdup ("Not a valid option for all "  \
+                                                "volumes");                    \
+                        goto label;                                            \
+                } else if (!_all && _ratio) {                                  \
+                        ret = -1;                                              \
+                        *op_errstr = gf_strdup ("Not a valid option for "      \
+                                                "single volume");              \
+                        goto label;                                            \
+                }                                                              \
+         } while (0)
+
 static struct list_head gd_op_sm_queue;
 pthread_mutex_t       gd_op_sm_lock;
 glusterd_op_info_t    opinfo = {{0},};
+
+int32_t
+glusterd_txn_opinfo_dict_init ()
+{
+        int32_t             ret    = -1;
+        xlator_t           *this   = NULL;
+        glusterd_conf_t    *priv   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        priv->glusterd_txn_opinfo = dict_new ();
+        if (!priv->glusterd_txn_opinfo) {
+                ret = -1;
+                goto out;
+        }
+
+        memset (priv->global_txn_id, '\0', sizeof(uuid_t));
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+glusterd_txn_opinfo_dict_fini ()
+{
+        xlator_t           *this   = NULL;
+        glusterd_conf_t    *priv   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (priv->glusterd_txn_opinfo)
+                dict_unref (priv->glusterd_txn_opinfo);
+}
+
+void
+glusterd_txn_opinfo_init (glusterd_op_info_t  *opinfo,
+                          glusterd_op_sm_state_info_t *state,
+                          glusterd_op_t *op,
+                          dict_t *op_ctx,
+                          rpcsvc_request_t *req)
+{
+        GF_ASSERT (opinfo);
+
+        if (state)
+                opinfo->state = *state;
+
+        if (op)
+                opinfo->op = *op;
+
+        if (op_ctx)
+                opinfo->op_ctx = dict_ref(op_ctx);
+        else
+                opinfo->op_ctx = NULL;
+
+        if (req)
+                opinfo->req = req;
+
+        return;
+}
+
+int32_t
+glusterd_generate_txn_id (dict_t *dict, uuid_t **txn_id)
+{
+        int32_t             ret      = -1;
+        glusterd_conf_t    *priv     = NULL;
+        xlator_t           *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (dict);
+
+        *txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!*txn_id)
+                goto out;
+
+        if (priv->op_version < GD_OP_VERSION_4)
+                uuid_copy (**txn_id, priv->global_txn_id);
+        else
+                uuid_generate (**txn_id);
+
+        ret = dict_set_bin (dict, "transaction_id",
+                            *txn_id, sizeof (uuid_t));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_DEBUG,
+                "Transaction_id = %s", uuid_utoa (**txn_id));
+out:
+        if (ret && *txn_id) {
+                GF_FREE (*txn_id);
+                *txn_id = NULL;
+        }
+
+        return ret;
+}
+
+int32_t
+glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+{
+        int32_t                   ret         = -1;
+        glusterd_txn_opinfo_obj  *opinfo_obj  = NULL;
+        glusterd_conf_t          *priv        = NULL;
+        xlator_t                 *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!txn_id || !opinfo) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Empty transaction id or opinfo received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(priv->glusterd_txn_opinfo,
+                           uuid_utoa (*txn_id),
+                           (void **) &opinfo_obj);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get transaction opinfo "
+                        "for transaction ID : %s",
+                        uuid_utoa (*txn_id));
+                goto out;
+        }
+
+        (*opinfo) = opinfo_obj->opinfo;
+
+        gf_log ("", GF_LOG_DEBUG,
+                "Successfully got opinfo for transaction ID : %s",
+                uuid_utoa (*txn_id));
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+{
+        int32_t                   ret        = -1;
+        glusterd_txn_opinfo_obj  *opinfo_obj = NULL;
+        glusterd_conf_t          *priv        = NULL;
+        xlator_t                 *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!txn_id) {
+                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(priv->glusterd_txn_opinfo,
+                           uuid_utoa (*txn_id),
+                           (void **) &opinfo_obj);
+        if (ret) {
+                opinfo_obj = GF_CALLOC (1, sizeof(glusterd_txn_opinfo_obj),
+                                        gf_common_mt_txn_opinfo_obj_t);
+                if (!opinfo_obj) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_bin(priv->glusterd_txn_opinfo,
+                                   uuid_utoa (*txn_id), opinfo_obj,
+                                   sizeof(glusterd_txn_opinfo_obj));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set opinfo for transaction ID : %s",
+                                uuid_utoa (*txn_id));
+                        goto out;
+                }
+        }
+
+        opinfo_obj->opinfo = (*opinfo);
+
+        gf_log ("", GF_LOG_DEBUG,
+                "Successfully set opinfo for transaction ID : %s",
+                uuid_utoa (*txn_id));
+        ret = 0;
+out:
+        if (ret)
+                if (opinfo_obj)
+                        GF_FREE (opinfo_obj);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_clear_txn_opinfo (uuid_t *txn_id)
+{
+        int32_t               ret         = -1;
+        glusterd_op_info_t    txn_op_info = {{0},};
+        glusterd_conf_t      *priv        = NULL;
+        xlator_t             *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!txn_id) {
+                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Transaction opinfo not found");
+                goto out;
+        }
+
+        if (txn_op_info.op_ctx)
+                dict_unref (txn_op_info.op_ctx);
+
+        dict_del(priv->glusterd_txn_opinfo, uuid_utoa (*txn_id));
+
+        gf_log ("", GF_LOG_DEBUG,
+                "Successfully cleared opinfo for transaction ID : %s",
+                uuid_utoa (*txn_id));
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
 static int glusterfs_port = GLUSTERD_DEFAULT_PORT;
 static char *glusterd_op_sm_state_names[] = {
         "Default",
@@ -96,6 +353,8 @@ static char *glusterd_op_sm_event_names[] = {
         "GD_OP_EVENT_INVALID"
 };
 
+extern struct volopt_map_entry glusterd_volopt_map[];
+
 char*
 glusterd_op_sm_state_name_get (int state)
 {
@@ -136,14 +395,48 @@ glusterd_is_volume_started (glusterd_volinfo_t  *volinfo)
 }
 
 static int
-glusterd_op_sm_inject_all_acc ()
+glusterd_op_sm_inject_all_acc (uuid_t *txn_id)
 {
         int32_t                 ret = -1;
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, txn_id, NULL);
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+static int
+glusterd_check_quota_cmd (char *key, char *value, char *errstr, size_t size)
+{
+        int                ret = -1;
+        gf_boolean_t       b   = _gf_false;
+
+        if ((strcmp (key, "quota") == 0) ||
+           (strcmp (key, "features.quota") == 0)) {
+                ret = gf_string2boolean (value, &b);
+                if (ret)
+                        goto out;
+                if (b) {
+                          snprintf (errstr, size," 'gluster "
+                                    "volume set <VOLNAME> %s %s' is "
+                                    "deprecated. Use 'gluster volume "
+                                    "quota <VOLNAME> enable' instead.",
+                                     key, value);
+                          ret = -1;
+                          goto out;
+                } else {
+                          snprintf (errstr, size, " 'gluster "
+                                    "volume set <VOLNAME> %s %s' is "
+                                    "deprecated. Use 'gluster volume "
+                                    "quota <VOLNAME> disable' instead.",
+                                     key, value);
+                          ret = -1;
+                          goto out;
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
 int
 glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
                                  gd1_mgmt_brick_op_req **req, dict_t *dict)
@@ -153,6 +446,10 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
         char                    *volname = NULL;
         char                    name[1024] = {0,};
         gf_xl_afr_op_t          heal_op = GF_AFR_OP_INVALID;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (op < GD_OP_MAX);
         GF_ASSERT (op > GD_OP_NONE);
@@ -164,10 +461,8 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
         case GD_OP_STOP_VOLUME:
                 brick_req = GF_CALLOC (1, sizeof (*brick_req),
                                        gf_gld_mt_mop_brick_req_t);
-                if (!brick_req) {
-                        gf_log ("", GF_LOG_ERROR, "Out of Memory");
+                if (!brick_req)
                         goto out;
-                }
                 brick_req->op = GLUSTERD_BRICK_TERMINATE;
                 brick_req->name = "";
         break;
@@ -175,10 +470,8 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
                 brick_req = GF_CALLOC (1, sizeof (*brick_req),
                                        gf_gld_mt_mop_brick_req_t);
 
-                if (!brick_req) {
-                        gf_log ("", GF_LOG_ERROR, "Out of Memory");
+                if (!brick_req)
                         goto out;
-                }
 
                 brick_req->op = GLUSTERD_BRICK_XLATOR_INFO;
                 brick_req->name = brickinfo->path;
@@ -203,10 +496,8 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
         {
                 brick_req = GF_CALLOC (1, sizeof (*brick_req),
                                        gf_gld_mt_mop_brick_req_t);
-                if (!brick_req) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Out of memory");
+                if (!brick_req)
                         goto out;
-                }
                 brick_req->op = GLUSTERD_BRICK_STATUS;
                 brick_req->name = "";
         }
@@ -226,13 +517,14 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
                 brick_req->name = gf_strdup (name);
 
                 break;
+
         default:
                 goto out;
         break;
         }
 
         ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
-                                           (size_t*)&brick_req->input.input_len);
+                                           &brick_req->input.input_len);
         if (ret)
                 goto out;
         *req = brick_req;
@@ -241,30 +533,151 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
 out:
         if (ret && brick_req)
                 GF_FREE (brick_req);
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+int
+glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+                                dict_t *dict)
+{
+        int                     ret = -1;
+        gd1_mgmt_brick_op_req   *brick_req = NULL;
+
+        GF_ASSERT (op < GD_OP_MAX);
+        GF_ASSERT (op > GD_OP_NONE);
+        GF_ASSERT (req);
+
+        switch (op) {
+        case GD_OP_PROFILE_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_NODE_PROFILE;
+                brick_req->name = "";
+
+                break;
+
+        case GD_OP_STATUS_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_NODE_STATUS;
+                brick_req->name = "";
+
+                break;
+
+        default:
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
+                                           &brick_req->input.input_len);
+
+        if (ret)
+                goto out;
+
+        *req = brick_req;
+        ret = 0;
+
+out:
+        if (ret && brick_req)
+                GF_FREE (brick_req);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_validate_quorum_options (xlator_t *this, char *fullkey, char *value,
+                                  char **op_errstr)
+{
+        int             ret = 0;
+        char            *key = NULL;
+        volume_option_t *opt = NULL;
+
+        if (!glusterd_is_quorum_option (fullkey))
+                goto out;
+        key = strchr (fullkey, '.');
+        if (key == NULL) {
+                ret = -1;
+                goto out;
+        }
+        key++;
+        opt = xlator_volume_option_get (this, key);
+        ret = xlator_option_validate (this, key, value, opt, op_errstr);
+out:
+        return ret;
+}
+
+static int
+glusterd_check_client_op_version_support (char *volname, uint32_t op_version,
+                                          char **op_errstr)
+{
+        int                     ret = 0;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        rpc_transport_t         *xprt = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        priv = this->private;
+        GF_ASSERT(priv);
+
+        pthread_mutex_lock (&priv->xprt_lock);
+        list_for_each_entry (xprt, &priv->xprt_list, list) {
+                if ((!strcmp(volname, xprt->peerinfo.volname)) &&
+                    ((op_version > xprt->peerinfo.max_op_version) ||
+                     (op_version < xprt->peerinfo.min_op_version))) {
+                        ret = -1;
+                        break;
+                }
+        }
+        pthread_mutex_unlock (&priv->xprt_lock);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "One or more clients "
+                        "don't support the required op-version");
+                ret = gf_asprintf (op_errstr, "One or more connected clients "
+                                   "cannot support the feature being set. "
+                                   "These clients need to be upgraded or "
+                                   "disconnected before running this command"
+                                   " again");
+                return -1;
+        }
+        return 0;
+}
 
 static int
 glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
 {
-        int                                      ret           = 0;
-        char                                    *volname       = NULL;
-        int                                      exists        = 0;
-        char                                    *key           = NULL;
-        char                                    *key_fixed     = NULL;
-        char                                    *value         = NULL;
-        char                                     str[100]      = {0, };
-        int                                      count         = 0;
-        int                                      dict_count    = 0;
-        char                                     errstr[2048]  = {0, };
-        glusterd_volinfo_t                      *volinfo       = NULL;
-        dict_t                                  *val_dict      = NULL;
-        gf_boolean_t                             global_opt    = _gf_false;
-        glusterd_volinfo_t                      *voliter       = NULL;
-        glusterd_conf_t                         *priv          = NULL;
-        xlator_t                                *this          = NULL;
+        int                             ret                     = -1;
+        char                            *volname                = NULL;
+        int                             exists                  = 0;
+        char                            *key                    = NULL;
+        char                            *key_fixed              = NULL;
+        char                            *value                  = NULL;
+        char                            str[100]                = {0, };
+        int                             count                   = 0;
+        int                             dict_count              = 0;
+        char                            errstr[2048]            = {0, };
+        glusterd_volinfo_t              *volinfo                = NULL;
+        dict_t                          *val_dict               = NULL;
+        gf_boolean_t                    global_opt              = _gf_false;
+        glusterd_volinfo_t              *voliter                = NULL;
+        glusterd_conf_t                 *priv                   = NULL;
+        xlator_t                        *this                   = NULL;
+        uint32_t                        new_op_version          = 0;
+        uint32_t                        local_new_op_version    = 0;
+        uint32_t                        key_op_version          = 0;
+        uint32_t                        local_key_op_version    = 0;
+        gf_boolean_t                    origin_glusterd         = _gf_true;
+        gf_boolean_t                    check_op_version        = _gf_true;
+        gf_boolean_t                    all_vol                 = _gf_false;
+        struct volopt_map_entry         *vme                    = NULL;
 
         GF_ASSERT (dict);
         this = THIS;
@@ -276,23 +689,54 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
         if (!val_dict)
                 goto out;
 
-        ret = dict_get_int32 (dict, "count", &dict_count);
+        /* Check if we can support the required op-version
+         * This check is not done on the originator glusterd. The originator
+         * glusterd sets this value.
+         */
+        origin_glusterd = is_origin_glusterd (dict);
+
+        if (!origin_glusterd) {
+                /* Check for v3.3.x origin glusterd */
+                check_op_version = dict_get_str_boolean (dict,
+                                                         "check-op-version",
+                                                         _gf_false);
+
+                if (check_op_version) {
+                        ret = dict_get_uint32 (dict, "new-op-version",
+                                               &new_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get new_op_version");
+                                goto out;
+                        }
+
+                        if ((new_op_version > GD_OP_VERSION_MAX) ||
+                            (new_op_version < GD_OP_VERSION_MIN)) {
+                                ret = -1;
+                                snprintf (errstr, sizeof (errstr),
+                                          "Required op_version (%d) is not "
+                                          "supported", new_op_version);
+                                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                                goto out;
+                        }
+                }
+        }
 
+        ret = dict_get_int32 (dict, "count", &dict_count);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Count(dict),not set in Volume-Set");
                 goto out;
         }
 
-        if ( dict_count == 0 ) {
-                /*No options would be specified of volume set help */
+        if (dict_count == 0) {
+        /*No options would be specified of volume set help */
                 if (dict_get (dict, "help" ))  {
                         ret = 0;
                         goto out;
                 }
 
                 if (dict_get (dict, "help-xml" )) {
-
 #if (HAVE_LIB_XML)
                         ret = 0;
                         goto out;
@@ -317,67 +761,156 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        exists = glusterd_check_volume_exists (volname);
-        if (!exists) {
-                snprintf (errstr, sizeof (errstr), "Volume %s does not exist",
-                          volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                *op_errstr = gf_strdup (errstr);
-                ret = -1;
-                goto out;
-        }
+        if (strcasecmp (volname, "all") != 0) {
+                exists = glusterd_check_volume_exists (volname);
+                if (!exists) {
+                        snprintf (errstr, sizeof (errstr),
+                                  FMTSTR_CHECK_VOL_EXISTS, volname);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        ret = -1;
+                        goto out;
+                }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to allocate memory");
-                goto out;
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                FMTSTR_CHECK_VOL_EXISTS, volname);
+                        goto out;
+                }
+
+                ret = glusterd_validate_volume_id (dict, volinfo);
+                if (ret)
+                        goto out;
+        } else {
+                all_vol = _gf_true;
         }
 
+        local_new_op_version = priv->op_version;
+
         for ( count = 1; ret != 1 ; count++ ) {
                 global_opt = _gf_false;
                 sprintf (str, "key%d", count);
                 ret = dict_get_str (dict, str, &key);
+                if (ret)
+                        break;
 
+                sprintf (str, "value%d", count);
+                ret = dict_get_str (dict, str, &value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid key,value pair in 'volume set'");
+                        ret = -1;
+                        goto out;
+                }
 
+                if (strcmp (key, "config.memory-accounting") == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "enabling memory accounting for volume %s",
+                                volname);
+                        ret = 0;
+                }
+
+                if (strcmp (key, "config.transport") == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "changing transport-type for volume %s",
+                                volname);
+                        ret = 0;
+                        /* if value is none of 'tcp/rdma/tcp,rdma' error out */
+                        if (!((strcasecmp (value, "rdma") == 0) ||
+                              (strcasecmp (value, "tcp") == 0) ||
+                              (strcasecmp (value, "tcp,rdma") == 0) ||
+                              (strcasecmp (value, "rdma,tcp") == 0))) {
+                                ret = snprintf (errstr, sizeof (errstr),
+                                                "transport-type %s does "
+                                                "not exist", value);
+                                /* lets not bother about above return value,
+                                   its a failure anyways */
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                ret = glusterd_check_quota_cmd (key, value, errstr, sizeof (errstr));
                 if (ret)
-                        break;
+                        goto out;
+
+                if (is_key_glusterd_hooks_friendly (key))
+                        continue;
+
+                for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
+                        if ((vme->validate_fn) &&
+                            ((!strcmp (key, vme->key)) ||
+                             (!strcmp (key, strchr (vme->key, '.') + 1)))) {
+                                ret = vme->validate_fn (dict, key, value,
+                                                        op_errstr);
+                                if (ret)
+                                        goto out;
+                                break;
+                        }
+                }
 
                 exists = glusterd_check_option_exists (key, &key_fixed);
                 if (exists == -1) {
                         ret = -1;
                         goto out;
                 }
+
                 if (!exists) {
                         gf_log (this->name, GF_LOG_ERROR,
-                                "Option with name: %s "
-                                "does not exist", key);
-                        ret = snprintf (errstr, 2048,
+                                "Option with name: %s does not exist", key);
+                        ret = snprintf (errstr, sizeof (errstr),
                                        "option : %s does not exist",
                                        key);
                         if (key_fixed)
-                                snprintf (errstr + ret, 2048 - ret,
+                                snprintf (errstr + ret, sizeof (errstr) - ret,
                                           "\nDid you mean %s?", key_fixed);
-                        *op_errstr = gf_strdup (errstr);
-                        ret = -1;
-                        goto out;
-                }
-
-                sprintf (str, "value%d", count);
-                ret = dict_get_str (dict, str, &value);
-
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid key,value pair in 'volume set'");
                         ret = -1;
                         goto out;
                 }
 
                 if (key_fixed)
                         key = key_fixed;
-
-                ret = glusterd_check_globaloption (key);
+                ALL_VOLUME_OPTION_CHECK (volname, key, ret, op_errstr, out);
+                ret = glusterd_validate_quorum_options (this, key, value,
+                                                        op_errstr);
                 if (ret)
+                        goto out;
+
+                local_key_op_version = glusterd_get_op_version_for_key (key);
+                if (local_key_op_version > local_new_op_version)
+                        local_new_op_version = local_key_op_version;
+
+                sprintf (str, "op-version%d", count);
+                if (origin_glusterd) {
+                        ret = dict_set_uint32 (dict, str, local_key_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set key-op-version in dict");
+                                goto out;
+                        }
+                } else if (check_op_version) {
+                        ret = dict_get_uint32 (dict, str, &key_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get key-op-version from"
+                                        " dict");
+                                goto out;
+                        }
+                        if (local_key_op_version != key_op_version) {
+                                ret = -1;
+                                snprintf (errstr, sizeof (errstr),
+                                          "option: %s op-version mismatch",
+                                          key);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s, required op-version = %"PRIu32", "
+                                        "available op-version = %"PRIu32,
+                                        errstr, key_op_version,
+                                        local_key_op_version);
+                                goto out;
+                        }
+                }
+
+                if (glusterd_check_globaloption (key))
                         global_opt = _gf_true;
 
                 ret = dict_set_str (val_dict, key, value);
@@ -390,9 +923,9 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                 }
 
                 *op_errstr = NULL;
-                if (!global_opt)
+                if (!global_opt && !all_vol)
                         ret = glusterd_validate_reconfopts (volinfo, val_dict, op_errstr);
-                else {
+                else if (!all_vol) {
                         voliter = NULL;
                         list_for_each_entry (voliter, &priv->volumes, vol_list) {
                                 ret = glusterd_validate_globalopts (voliter, val_dict, op_errstr);
@@ -402,8 +935,9 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                 }
 
                 if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Could not create temp "
-                                "volfile, some option failed: %s", *op_errstr);
+                        gf_log (this->name, GF_LOG_ERROR, "Could not create "
+                                "temp volfile, some option failed: %s",
+                                *op_errstr);
                         goto out;
                 }
                 dict_del (val_dict, key);
@@ -414,6 +948,35 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                 }
         }
 
+        // Check if all the connected clients support the new op-version
+        ret = glusterd_check_client_op_version_support (volname,
+                                                        local_new_op_version,
+                                                        op_errstr);
+        if (ret)
+                goto out;
+
+        if (origin_glusterd) {
+                ret = dict_set_uint32 (dict, "new-op-version",
+                                       local_new_op_version);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set new-op-version in dict");
+                        goto out;
+                }
+                /* Set this value in dict so other peers know to check for
+                 * op-version. This is a hack for 3.3.x compatibility
+                 *
+                 * TODO: Remove this and the other places this is referred once
+                 * 3.3.x compatibility is not required
+                 */
+                ret = dict_set_uint32 (dict, "check-op-version",
+                                       _gf_true);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set check-op-version in dict");
+                        goto out;
+                }
+        }
 
         ret = 0;
 
@@ -421,8 +984,9 @@ out:
         if (val_dict)
                 dict_unref (val_dict);
 
-        if (key_fixed)
-                GF_FREE (key_fixed);
+        GF_FREE (key_fixed);
+        if (errstr[0] != '\0')
+                *op_errstr = gf_strdup (errstr);
 
         if (ret) {
                 if (!(*op_errstr)) {
@@ -447,28 +1011,42 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
         char                                    msg[2048]      = {0};
         char                                    *key = NULL;
         char                                    *key_fixed = NULL;
+        glusterd_volinfo_t                      *volinfo       = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
-        exists = glusterd_check_volume_exists (volname);
+        if (strcasecmp (volname, "all") != 0) {
+                exists = glusterd_check_volume_exists (volname);
+                if (!exists) {
+                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
+                                  volname);
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
+                                  volname);
+                        goto out;
+                }
 
-        if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not "
-                          "exist", volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
+                ret = glusterd_validate_volume_id (dict, volinfo);
+                if (ret)
+                        goto out;
         }
 
         ret = dict_get_str (dict, "key", &key);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get option key");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get option key");
                 goto out;
         }
         if (strcmp(key, "all")) {
@@ -478,24 +1056,30 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
                         goto out;
                 }
                 if (!exists) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Option %s does not exist", key);
-                        ret = snprintf (msg, 2048,
+                        ret = snprintf (msg, sizeof (msg),
                                         "Option %s does not exist", key);
                         if (key_fixed)
-                                snprintf (msg + ret, 2048 - ret,
+                                snprintf (msg + ret, sizeof (msg) - ret,
                                           "\nDid you mean %s?", key_fixed);
-                        *op_errstr = gf_strdup (msg);
                         ret = -1;
                         goto out;
+                } else if (exists > 0) {
+                        if (key_fixed)
+                                key = key_fixed;
+                        ALL_VOLUME_OPTION_CHECK (volname, key, ret,
+                                                 op_errstr, out);
                 }
         }
 
 out:
-        if (key_fixed)
-                GF_FREE (key_fixed);
+        GF_FREE (key_fixed);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -511,6 +1095,7 @@ glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
         gf_boolean_t                            exists = _gf_false;
         glusterd_peerinfo_t                     *peerinfo = NULL;
         char                                    msg[2048] = {0,};
+        glusterd_volinfo_t                      *volinfo  = NULL;
 
         ret = dict_get_str (dict, "hostname", &hostname);
         if (ret) {
@@ -520,8 +1105,26 @@ glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        ret = glusterd_is_local_addr (hostname);
-        if (ret) {
+        if (gf_is_local_addr (hostname)) {
+                //volname is not present in case of sync all
+                ret = dict_get_str (dict, "volname", &volname);
+                if (!ret) {
+                        exists = glusterd_check_volume_exists (volname);
+                        if (!exists) {
+                                snprintf (msg, sizeof (msg), "Volume %s "
+                                         "does not exist", volname);
+                                *op_errstr = gf_strdup (msg);
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret)
+                                goto out;
+
+                } else {
+                        ret = 0;
+                }
+         } else {
                 ret = glusterd_friend_find (NULL, hostname, &peerinfo);
                 if (ret) {
                         snprintf (msg, sizeof (msg), "%s, is not a friend",
@@ -537,22 +1140,7 @@ glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
                         ret = -1;
                         goto out;
                 }
-        } else {
 
-                //volname is not present in case of sync all
-                ret = dict_get_str (dict, "volname", &volname);
-                if (!ret) {
-                        exists = glusterd_check_volume_exists (volname);
-                        if (!exists) {
-                                snprintf (msg, sizeof (msg), "Volume %s "
-                                         "does not exist", volname);
-                                *op_errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        ret = 0;
-                }
         }
 
 out:
@@ -572,8 +1160,10 @@ glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
         xlator_t              *this           = NULL;
         glusterd_conf_t       *priv           = NULL;
         glusterd_brickinfo_t  *brickinfo      = NULL;
-        glusterd_brickinfo_t  *tmpbrickinfo   = NULL;
         glusterd_volinfo_t    *volinfo        = NULL;
+        dict_t                *vol_opts       = NULL;
+        gf_boolean_t           nfs_disabled   = _gf_false;
+        gf_boolean_t           shd_enabled    = _gf_true;
 
         GF_ASSERT (dict);
         this = THIS;
@@ -588,46 +1178,88 @@ glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
         if (cmd & GF_CLI_STATUS_ALL)
                 goto out;
 
+        if ((cmd & GF_CLI_STATUS_QUOTAD) &&
+            (priv->op_version == GD_OP_VERSION_MIN)) {
+                snprintf (msg, sizeof (msg), "The cluster is operating at "
+                          "version 1. Getting the status of quotad is not "
+                          "allowed in this state.");
+                ret = -1;
+                goto out;
+        }
+
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                snprintf (msg, sizeof(msg), "Volume %s does not exist", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                snprintf (msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
                 ret = -1;
                 goto out;
         }
 
-        if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
-                ret = dict_get_str (dict, "brick", &brick);
-                if (ret)
-                        goto out;
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
 
-                ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "%s is not a brick",
-                                  brick);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+        ret = glusterd_is_volume_started (volinfo);
+        if (!ret) {
+                snprintf (msg, sizeof (msg), "Volume %s is not started",
+                          volname);
+                ret = -1;
+                goto out;
+        }
+
+        vol_opts = volinfo->dict;
+
+        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                nfs_disabled = dict_get_str_boolean (vol_opts, "nfs.disable",
+                                                     _gf_false);
+                if (nfs_disabled) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "NFS server is disabled for volume %s",
+                                  volname);
+                        goto out;
+                }
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                if (!glusterd_is_volume_replicate (volinfo)) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "Volume %s is not of type replicate",
+                                  volname);
                         goto out;
                 }
 
-                ret = glusterd_volume_brickinfo_get (NULL,
-                                                     brickinfo->hostname,
-                                                     brickinfo->path,
-                                                     volinfo,
-                                                     &tmpbrickinfo,
-                                                     GF_PATH_COMPLETE);
+                shd_enabled = dict_get_str_boolean (vol_opts,
+                                                    "cluster.self-heal-daemon",
+                                                    _gf_true);
+                if (!shd_enabled) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "Self-heal Daemon is disabled for volume %s",
+                                  volname);
+                        goto out;
+                }
+        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+                if (!glusterd_is_volume_quota_enabled (volinfo)) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg), "Volume %s does not have "
+                                  "quota enabled", volname);
+                        goto out;
+                }
+        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+                ret = dict_get_str (dict, "brick", &brick);
+                if (ret)
+                        goto out;
 
+                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                              &brickinfo);
                 if (ret) {
                         snprintf (msg, sizeof(msg), "No brick %s in"
                                   " volume %s", brick, volname);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-
                         ret = -1;
                         goto out;
                 }
@@ -643,7 +1275,7 @@ glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
                         *op_errstr = gf_strdup ("Validation Failed for Status");
         }
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning: %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning: %d", ret);
         return ret;
 }
 
@@ -694,6 +1326,10 @@ glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         ret = dict_get_int32 (dict, "op", &stats_op);
         if (ret) {
                 snprintf (msg, sizeof (msg), "Volume profile op get failed");
@@ -740,60 +1376,82 @@ out:
 }
 
 
-void
+static int
 _delete_reconfig_opt (dict_t *this, char *key, data_t *value, void *data)
 {
-        int             exists = 0;
-        int32_t         is_force = 0;
+        int32_t        *is_force = 0;
 
         GF_ASSERT (data);
-        is_force = *((int32_t*)data);
-        exists = glusterd_check_option_exists(key, NULL);
-
-        if (exists != 1)
-                goto out;
-
-        if ((!is_force) &&
-            (_gf_true == glusterd_check_voloption_flags (key,
-                                                         OPT_FLAG_FORCE)))
-                goto out;
+        is_force = (int32_t*)data;
+
+        if (*is_force != 1) {
+                if (_gf_true == glusterd_check_voloption_flags (key,
+                                                         OPT_FLAG_FORCE)) {
+                /* indicate to caller that we don't set the option
+                 * due to being protected
+                 */
+                        *is_force = *is_force | GD_OP_PROTECTED;
+                        goto out;
+                } else {
+                        *is_force = *is_force | GD_OP_UNPROTECTED;
+                }
+        }
 
         gf_log ("", GF_LOG_DEBUG, "deleting dict with key=%s,value=%s",
                 key, value->data);
         dict_del (this, key);
 out:
-        return;
+        return 0;
 }
 
-int
+static int
+_delete_reconfig_global_opt (dict_t *this, char *key, data_t *value, void *data)
+{
+        int32_t        *is_force = 0;
+
+        GF_ASSERT (data);
+        is_force = (int32_t*)data;
+
+        if (strcmp (GLUSTERD_GLOBAL_OPT_VERSION, key) == 0)
+                goto out;
+
+        _delete_reconfig_opt (this, key, value, data);
+out:
+        return 0;
+}
+
+static int
 glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
-                        int32_t is_force)
+                        int32_t *is_force)
 {
         int                      ret = 0;
         data_t                  *value = NULL;
+        char                    *key_fixed = NULL;
+        xlator_t                *this = NULL;
 
-        gf_log ("", GF_LOG_DEBUG, "Received volume set reset command");
-
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (volinfo->dict);
         GF_ASSERT (key);
 
         if (!strncmp(key, "all", 3))
-                dict_foreach (volinfo->dict, _delete_reconfig_opt, &is_force);
+                dict_foreach (volinfo->dict, _delete_reconfig_opt, is_force);
         else {
                 value = dict_get (volinfo->dict, key);
                 if (!value) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Could not get value");
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "no value set for option %s", key);
                         goto out;
                 }
-                _delete_reconfig_opt (volinfo->dict, key, value, &is_force);
+                _delete_reconfig_opt (volinfo->dict, key, value, is_force);
         }
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        gd_update_volume_op_versions (volinfo);
 
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to create volfile for"
-                        " 'volume set'");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create volfile for"
+                        " 'volume reset'");
                 ret = -1;
                 goto out;
         }
@@ -811,23 +1469,120 @@ glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        GF_FREE (key_fixed);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+static int
+glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
+{
+        char            *key            = NULL;
+        char            *key_fixed      = NULL;
+        int             ret             = -1;
+        int32_t         is_force        = 0;
+        glusterd_conf_t *conf           = NULL;
+        dict_t          *dup_opt        = NULL;
+        gf_boolean_t    all             = _gf_false;
+        char            *next_version   = NULL;
+        gf_boolean_t    quorum_action   = _gf_false;
+
+        conf = this->private;
+        ret = dict_get_str (dict, "key", &key);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get key");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "force", &is_force);
+        if (ret)
+                is_force = 0;
+
+        if (strcmp (key, "all")) {
+                ret = glusterd_check_option_exists (key, &key_fixed);
+                if (ret <= 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Option %s does not "
+                                "exist", key);
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                all = _gf_true;
+        }
+
+        if (key_fixed)
+                key = key_fixed;
+
+        ret = -1;
+        dup_opt = dict_new ();
+        if (!dup_opt)
+                goto out;
+        if (!all) {
+                dict_copy (conf->opts, dup_opt);
+                dict_del (dup_opt, key);
+        }
+        ret = glusterd_get_next_global_opt_version_str (conf->opts,
+                                                        &next_version);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_options (this, dup_opt);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_quorum_changed (conf->opts, key, NULL))
+                quorum_action = _gf_true;
+
+        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                               next_version);
+        if (ret)
+                goto out;
+        else
+                next_version = NULL;
+
+        if (!all) {
+                dict_del (conf->opts, key);
+        } else {
+                dict_foreach (conf->opts, _delete_reconfig_global_opt,
+                              &is_force);
+        }
+out:
+        GF_FREE (key_fixed);
+        if (dup_opt)
+                dict_unref (dup_opt);
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        GF_FREE (next_version);
+        return ret;
+}
 
 static int
-glusterd_op_reset_volume (dict_t *dict)
+glusterd_op_reset_volume (dict_t *dict, char **op_rspstr)
 {
         glusterd_volinfo_t      *volinfo    = NULL;
         int                     ret         = -1;
         char                    *volname    = NULL;
         char                    *key        = NULL;
+        char                    *key_fixed  = NULL;
         int32_t                 is_force    = 0;
+        gf_boolean_t            quorum_action = _gf_false;
+        xlator_t                *this         = NULL;
 
+        this = THIS;
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name " );
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name" );
+                goto out;
+        }
+
+        if (strcasecmp (volname, "all") == 0) {
+                ret = glusterd_op_reset_all_volume_options (this, dict);
                 goto out;
         }
 
@@ -837,32 +1592,64 @@ glusterd_op_reset_volume (dict_t *dict)
 
         ret = dict_get_str (dict, "key", &key);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get option key");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get option key");
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
         }
 
-        ret = glusterd_options_reset (volinfo, key, is_force);
+        if (strcmp (key, "all") &&
+            glusterd_check_option_exists (key, &key_fixed) != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "volinfo dict inconsistency: option %s not found",
+                        key);
+                ret = -1;
+                goto out;
+        }
+        if (key_fixed)
+                key = key_fixed;
+
+        if (glusterd_is_quorum_changed (volinfo->dict, key, NULL))
+                quorum_action = _gf_true;
+
+        ret = glusterd_options_reset (volinfo, key, &is_force);
+        if (ret == -1) {
+                gf_asprintf(op_rspstr, "Volume reset : failed");
+        } else if (is_force & GD_OP_PROTECTED) {
+                if (is_force & GD_OP_UNPROTECTED) {
+                        gf_asprintf (op_rspstr, "All unprotected fields were"
+                                     " reset. To reset the protected fields,"
+                                     " use 'force'.");
+                } else {
+                        ret = -1;
+                        gf_asprintf (op_rspstr, "'%s' is protected. To reset"
+                                     " use 'force'.", key);
+                }
+        }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "'volume reset' returning %d", ret);
-        return ret;
+        GF_FREE (key_fixed);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
 
+        gf_log (this->name, GF_LOG_DEBUG, "'volume reset' returning %d", ret);
+        return ret;
 }
 
-
 int
 glusterd_stop_bricks (glusterd_volinfo_t *volinfo)
 {
         glusterd_brickinfo_t                    *brickinfo = NULL;
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_brick_stop (volinfo, brickinfo))
+            /*TODO: Need to change @del_brick in brick_stop to _gf_true
+             * once we enable synctask in peer rpc prog */
+                if (glusterd_brick_stop (volinfo, brickinfo, _gf_false))
                         return -1;
         }
 
@@ -875,7 +1662,7 @@ glusterd_start_bricks (glusterd_volinfo_t *volinfo)
         glusterd_brickinfo_t                    *brickinfo = NULL;
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_brick_start (volinfo, brickinfo))
+                if (glusterd_brick_start (volinfo, brickinfo, _gf_false))
                         return -1;
         }
 
@@ -883,21 +1670,91 @@ glusterd_start_bricks (glusterd_volinfo_t *volinfo)
 }
 
 static int
-glusterd_volset_help (dict_t *dict)
+glusterd_op_set_all_volume_options (xlator_t *this, dict_t *dict)
 {
-        int                     ret = -1;
-        gf_boolean_t            xml_out = _gf_false;
+        char            *key            = NULL;
+        char            *key_fixed      = NULL;
+        char            *value          = NULL;
+        char            *dup_value      = NULL;
+        int             ret             = -1;
+        glusterd_conf_t *conf           = NULL;
+        dict_t          *dup_opt        = NULL;
+        char            *next_version   = NULL;
+        gf_boolean_t    quorum_action   = _gf_false;
+
+        conf = this->private;
+        ret = dict_get_str (dict, "key1", &key);
+        if (ret)
+                goto out;
+
+        ret = dict_get_str (dict, "value1", &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "invalid key,value pair in 'volume set'");
+                goto out;
+        }
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid key %s", key);
+                ret = -1;
+                goto out;
+        }
+
+        if (key_fixed)
+                key = key_fixed;
+
+        ret = -1;
+        dup_opt = dict_new ();
+        if (!dup_opt)
+                goto out;
+        dict_copy (conf->opts, dup_opt);
+        ret = dict_set_str (dup_opt, key, value);
+        if (ret)
+                goto out;
+
+        ret = glusterd_get_next_global_opt_version_str (conf->opts,
+                                                        &next_version);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_options (this, dup_opt);
+        if (ret)
+                goto out;
 
-        if (dict_get (dict, "help" ))
-                xml_out = _gf_false;
-        else if (dict_get (dict, "help-xml" ))
-                xml_out = _gf_true;
+        if (glusterd_is_quorum_changed (conf->opts, key, value))
+                quorum_action = _gf_true;
+
+        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                               next_version);
+        if (ret)
+                goto out;
         else
+                next_version = NULL;
+
+        dup_value = gf_strdup (value);
+        if (!dup_value)
                 goto out;
 
-        ret = glusterd_get_volopt_content (xml_out);
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        ret = dict_set_dynstr (conf->opts, key, dup_value);
+        if (ret)
+                goto out;
+        else
+                dup_value = NULL; /* Protect the allocation from GF_FREE */
+
+out:
+        GF_FREE (dup_value);
+        GF_FREE (key_fixed);
+        if (dup_opt)
+                dict_unref (dup_opt);
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        GF_FREE (next_version);
         return ret;
 }
 
@@ -914,9 +1771,14 @@ glusterd_op_set_volume (dict_t *dict)
         char                                    *key_fixed = NULL;
         char                                    *value = NULL;
         char                                     str[50] = {0, };
+        char                                    *op_errstr = NULL;
         gf_boolean_t                             global_opt    = _gf_false;
+        gf_boolean_t                             global_opts_set = _gf_false;
         glusterd_volinfo_t                      *voliter = NULL;
         int32_t                                  dict_count = 0;
+        gf_boolean_t                             check_op_version = _gf_false;
+        uint32_t                                 new_op_version = 0;
+        gf_boolean_t                            quorum_action  = _gf_false;
 
         this = THIS;
         GF_ASSERT (this);
@@ -931,12 +1793,16 @@ glusterd_op_set_volume (dict_t *dict)
         }
 
         if (dict_count == 0) {
-                ret = glusterd_volset_help (dict);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Volume set"
-                                " help internal error");
+                ret = glusterd_volset_help (NULL, &op_errstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s", 
+                                       (op_errstr)? op_errstr:
+                                       "Volume set help internal error");
+                }
+
+                GF_FREE(op_errstr);
                 goto out;
-        }
+         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
@@ -944,42 +1810,84 @@ glusterd_op_set_volume (dict_t *dict)
                 goto out;
         }
 
+        if (strcasecmp (volname, "all") == 0) {
+                ret = glusterd_op_set_all_volume_options (this, dict);
+                goto out;
+        }
+
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
         }
 
+        // TODO: Remove this once v3.3 compatability is not required
+        check_op_version = dict_get_str_boolean (dict, "check-op-version",
+                                                 _gf_false);
+
+        if (check_op_version) {
+                ret = dict_get_uint32 (dict, "new-op-version", &new_op_version);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get new op-version from dict");
+                        goto out;
+                }
+        }
+
         for (count = 1; ret != -1 ; count++) {
 
-                global_opt = _gf_false;
                 sprintf (str, "key%d", count);
                 ret = dict_get_str (dict, str, &key);
-
                 if (ret)
                         break;
 
-                if (!ret) {
+                sprintf (str, "value%d", count);
+                ret = dict_get_str (dict, str, &value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid key,value pair in 'volume set'");
+                        ret = -1;
+                        goto out;
+                }
+
+                if (strcmp (key, "config.memory-accounting") == 0) {
+                        ret = gf_string2boolean (value,
+                                                 &volinfo->memory_accounting);
+                }
+
+                if (strcmp (key, "config.transport") == 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "changing transport-type for volume %s to %s",
+                                volname, value);
+                        ret = 0;
+                        if (strcasecmp (value, "rdma") == 0) {
+                                volinfo->transport_type = GF_TRANSPORT_RDMA;
+                        } else if (strcasecmp (value, "tcp") == 0) {
+                                volinfo->transport_type = GF_TRANSPORT_TCP;
+                        } else if ((strcasecmp (value, "tcp,rdma") == 0) ||
+                                   (strcasecmp (value, "rdma,tcp") == 0)) {
+                                volinfo->transport_type =
+                                        GF_TRANSPORT_BOTH_TCP_RDMA;
+                        } else {
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                if (!is_key_glusterd_hooks_friendly (key)) {
                         ret = glusterd_check_option_exists (key, &key_fixed);
                         GF_ASSERT (ret);
-                        if (ret == -1) {
+                        if (ret <= 0) {
                                 key_fixed = NULL;
                                 goto out;
                         }
-                        ret = 0;
                 }
 
-                ret = glusterd_check_globaloption (key);
-                if (ret)
+                global_opt = _gf_false;
+                if (glusterd_check_globaloption (key)) {
                         global_opt = _gf_true;
-
-                sprintf (str, "value%d", count);
-                ret = dict_get_str (dict, str, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid key,value pair in 'volume set'");
-                        ret = -1;
-                        goto out;
+                        global_opts_set = _gf_true;
                 }
 
                 if (!global_opt)
@@ -995,6 +1903,9 @@ glusterd_op_set_volume (dict_t *dict)
                 if (key_fixed)
                         key = key_fixed;
 
+                if (glusterd_is_quorum_changed (volinfo->dict, key, value))
+                        quorum_action = _gf_true;
+
                 if (global_opt) {
                        list_for_each_entry (voliter, &priv->volumes, vol_list) {
                                value = gf_strdup (value);
@@ -1014,13 +1925,27 @@ glusterd_op_set_volume (dict_t *dict)
                 }
         }
 
-        if ( count == 1 ) {
+        if (count == 1) {
                 gf_log (this->name, GF_LOG_ERROR, "No options received ");
                 ret = -1;
                 goto out;
         }
 
-        if (!global_opt) {
+        /* Update the cluster op-version before regenerating volfiles so that
+         * correct volfiles are generated
+         */
+        if (new_op_version > priv->op_version) {
+                priv->op_version = new_op_version;
+                ret = glusterd_store_global_info (this);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store op-version");
+                        goto out;
+                }
+        }
+
+        if (!global_opts_set) {
+                gd_update_volume_op_versions (volinfo);
                 ret = glusterd_create_volfiles_and_notify_services (volinfo);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR,
@@ -1046,6 +1971,7 @@ glusterd_op_set_volume (dict_t *dict)
         } else {
                 list_for_each_entry (voliter, &priv->volumes, vol_list) {
                         volinfo = voliter;
+                        gd_update_volume_op_versions (volinfo);
                         ret = glusterd_create_volfiles_and_notify_services (volinfo);
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR,
@@ -1071,11 +1997,11 @@ glusterd_op_set_volume (dict_t *dict)
                 }
         }
 
-        ret = 0;
  out:
-        if (key_fixed)
-                GF_FREE (key_fixed);
+        GF_FREE (key_fixed);
         gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
         return ret;
 }
 
@@ -1107,7 +2033,7 @@ glusterd_op_sync_volume (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
-        if (glusterd_is_local_addr (hostname)) {
+        if (!gf_is_local_addr (hostname)) {
                 ret = 0;
                 goto out;
         }
@@ -1278,12 +2204,249 @@ out:
 }
 
 static int
+_add_brick_name_to_dict (dict_t *dict, char *key, glusterd_brickinfo_t *brick)
+{
+        int     ret = -1;
+        char    tmp[1024] = {0,};
+        char    *brickname = NULL;
+        xlator_t *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+        GF_ASSERT (brick);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        snprintf (tmp, sizeof (tmp), "%s:%s", brick->hostname, brick->path);
+        brickname = gf_strdup (tmp);
+        if (!brickname) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to dup brick name");
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dict, key, brickname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to add brick name to dict");
+                goto out;
+        }
+        brickname = NULL;
+out:
+        if (brickname)
+                GF_FREE (brickname);
+        return ret;
+}
+
+static int
+_add_remove_bricks_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
+                            char *prefix)
+{
+        int             ret = -1;
+        int             count = 0;
+        int             i = 0;
+        char            brick_key[1024] = {0,};
+        char            dict_key[1024] ={0,};
+        char            *brick = NULL;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+        GF_ASSERT (prefix);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (volinfo->rebal.dict, "count", &count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get brick count");
+                goto out;
+        }
+
+        snprintf (dict_key, sizeof (dict_key), "%s.count", prefix);
+        ret = dict_set_int32 (dict, dict_key, count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set brick count in dict");
+                goto out;
+        }
+
+        for (i = 1; i <= count; i++) {
+                memset (brick_key, 0, sizeof (brick_key));
+                snprintf (brick_key, sizeof (brick_key), "brick%d", i);
+
+                ret = dict_get_str (volinfo->rebal.dict, brick_key, &brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s", brick_key);
+                        goto out;
+                }
+
+                memset (dict_key, 0, sizeof (dict_key));
+                snprintf (dict_key, sizeof (dict_key), "%s.%s", prefix,
+                          brick_key);
+                ret = dict_set_str (dict, dict_key, brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add brick to dict");
+                        goto out;
+                }
+                brick = NULL;
+        }
+
+out:
+        return ret;
+}
+
+/* This adds the respective task-id and all available parameters of a task into
+ * a dictionary
+ */
+static int
+_add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
+{
+
+        int             ret = -1;
+        char            key[128] = {0,};
+        char            *uuid_str = NULL;
+        int             status = 0;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        switch (op) {
+        case GD_OP_REMOVE_BRICK:
+                snprintf (key, sizeof (key), "task%d", index);
+                ret = _add_remove_bricks_to_dict (dict, volinfo, key);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add remove bricks to dict");
+                        goto out;
+                }
+        case GD_OP_REBALANCE:
+                uuid_str = gf_strdup (uuid_utoa (volinfo->rebal.rebalance_id));
+                status = volinfo->rebal.defrag_status;
+                break;
+
+        case GD_OP_REPLACE_BRICK:
+                snprintf (key, sizeof (key), "task%d.src-brick", index);
+                ret = _add_brick_name_to_dict (dict, key,
+                                               volinfo->rep_brick.src_brick);
+                if (ret)
+                        goto out;
+                memset (key, 0, sizeof (key));
+
+                snprintf (key, sizeof (key), "task%d.dst-brick", index);
+                ret = _add_brick_name_to_dict (dict, key,
+                                               volinfo->rep_brick.dst_brick);
+                if (ret)
+                        goto out;
+                memset (key, 0, sizeof (key));
+
+                uuid_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
+                status = volinfo->rep_brick.rb_status;
+                break;
+
+        default:
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s operation doesn't have a"
+                        " task_id", gd_op_list[op]);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "task%d.type", index);
+        ret = dict_set_str (dict, key, (char *)gd_op_list[op]);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task type in dict");
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "task%d.id", index);
+
+        if (!uuid_str)
+                goto out;
+        ret = dict_set_dynstr (dict, key, uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task id in dict");
+                goto out;
+        }
+        uuid_str = NULL;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "task%d.status", index);
+        ret = dict_set_int32 (dict, key, status);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task status in dict");
+                goto out;
+        }
+
+out:
+        if (uuid_str)
+                GF_FREE (uuid_str);
+        return ret;
+}
+
+static int
+glusterd_aggregate_task_status (dict_t *rsp_dict, glusterd_volinfo_t *volinfo)
+{
+        int        ret   = -1;
+        int        tasks = 0;
+        xlator_t  *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                ret = _add_task_to_dict (rsp_dict, volinfo, volinfo->rebal.op,
+                                         tasks);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add task details to dict");
+                        goto out;
+                }
+                tasks++;
+        }
+
+        if (!uuid_is_null (volinfo->rep_brick.rb_id)) {
+                ret = _add_task_to_dict (rsp_dict, volinfo, GD_OP_REPLACE_BRICK,
+                                         tasks);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add task details to dict");
+                        goto out;
+                }
+                tasks++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "tasks", tasks);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting tasks count in dict");
+                goto out;
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
 glusterd_op_status_volume (dict_t *dict, char **op_errstr,
                            dict_t *rsp_dict)
 {
         int                     ret             = -1;
-        int                     brick_count     = 0;
+        int                     node_count      = 0;
         int                     brick_index     = -1;
+        int                     other_count     = 0;
+        int                     other_index     = 0;
         uint32_t                cmd             = 0;
         char                   *volname         = NULL;
         char                   *brick           = NULL;
@@ -1291,6 +2454,10 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
         glusterd_volinfo_t     *volinfo         = NULL;
         glusterd_brickinfo_t   *brickinfo       = NULL;
         glusterd_conf_t        *priv            = NULL;
+        dict_t                 *vol_opts        = NULL;
+        gf_boolean_t            nfs_disabled    = _gf_false;
+        gf_boolean_t            shd_enabled     = _gf_true;
+        gf_boolean_t            origin_glusterd = _gf_false;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1300,23 +2467,21 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
 
         GF_ASSERT (dict);
 
+        origin_glusterd = is_origin_glusterd (dict);
+
         ret = dict_get_uint32 (dict, "cmd", &cmd);
         if (ret)
                 goto out;
 
-        if (!rsp_dict) {
-                //this should happen only on source
+        if (origin_glusterd) {
                 ret = 0;
-                rsp_dict = glusterd_op_get_ctx ();
-
                 if ((cmd & GF_CLI_STATUS_ALL)) {
                         ret = glusterd_get_all_volnames (rsp_dict);
                         if (ret)
-                                gf_log (THIS->name, GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "failed to get all volume "
                                         "names for status");
                 }
-
         }
 
         ret = dict_set_uint32 (rsp_dict, "cmd", cmd);
@@ -1332,37 +2497,65 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Volume with name: %s "
+                gf_log (this->name, GF_LOG_ERROR, "Volume with name: %s "
                         "does not exist", volname);
                 goto out;
         }
+        vol_opts = volinfo->dict;
+
+        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                ret = glusterd_add_node_to_dict ("nfs", rsp_dict, 0, vol_opts);
+                if (ret)
+                        goto out;
+                other_count++;
+                node_count++;
+
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                ret = glusterd_add_node_to_dict ("glustershd", rsp_dict, 0,
+                                                 vol_opts);
+                if (ret)
+                        goto out;
+                other_count++;
+                node_count++;
+
+        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+                ret = glusterd_add_node_to_dict ("quotad", rsp_dict, 0,
+                                                 vol_opts);
+                if (ret)
+                        goto out;
+                other_count++;
+                node_count++;
 
-        if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
                 ret = dict_get_str (dict, "brick", &brick);
                 if (ret)
                         goto out;
 
                 ret = glusterd_volume_brickinfo_get_by_brick (brick,
                                                               volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_COMPLETE);
+                                                              &brickinfo);
                 if (ret)
                         goto out;
 
-                if (uuid_compare (brickinfo->uuid, priv->uuid))
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
                         goto out;
 
-                glusterd_add_brick_to_dict (volinfo, brickinfo, rsp_dict, 0);
+                glusterd_add_brick_to_dict (volinfo, brickinfo, rsp_dict,
+                                            ++brick_index);
                 if (cmd & GF_CLI_STATUS_DETAIL)
                         glusterd_add_brick_detail_to_dict (volinfo, brickinfo,
-                                                           rsp_dict, 0);
-                ret = dict_set_int32 (rsp_dict, "count", 1);
+                                                           rsp_dict,
+                                                           brick_index);
+                node_count++;
 
+        } else if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+                ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
                 goto out;
+
         } else {
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         brick_index++;
-                        if (uuid_compare (brickinfo->uuid, priv->uuid))
+                        if (uuid_compare (brickinfo->uuid, MY_UUID))
                                 continue;
 
                         glusterd_add_brick_to_dict (volinfo, brickinfo,
@@ -1374,14 +2567,88 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
                                                                    rsp_dict,
                                                                    brick_index);
                         }
-                        brick_count++;
+                        node_count++;
+                }
+
+                if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
+                        other_index = brick_index + 1;
+
+                        nfs_disabled = dict_get_str_boolean (vol_opts,
+                                                             "nfs.disable",
+                                                             _gf_false);
+                        if (!nfs_disabled) {
+                                ret = glusterd_add_node_to_dict ("nfs",
+                                                                 rsp_dict,
+                                                                 other_index,
+                                                                 vol_opts);
+                                if (ret)
+                                        goto out;
+                                other_index++;
+                                other_count++;
+                                node_count++;
+                        }
+
+                        shd_enabled = dict_get_str_boolean
+                                        (vol_opts, "cluster.self-heal-daemon",
+                                         _gf_true);
+                        if (glusterd_is_volume_replicate (volinfo)
+                            && shd_enabled) {
+                                ret = glusterd_add_node_to_dict ("glustershd",
+                                                                 rsp_dict,
+                                                                 other_index,
+                                                                 vol_opts);
+                                if (ret)
+                                        goto out;
+                                other_count++;
+                                node_count++;
+                                other_index++;
+                        }
+                        if (glusterd_is_volume_quota_enabled (volinfo)) {
+                                ret = glusterd_add_node_to_dict ("quotad",
+                                                                 rsp_dict,
+                                                                 other_index,
+                                                                 vol_opts);
+                                if (ret)
+                                        goto out;
+                                other_count++;
+                                node_count++;
+                        }
                 }
         }
 
-        ret = dict_set_int32 (rsp_dict, "count", brick_count);
+        ret = dict_set_int32 (rsp_dict, "brick-index-max", brick_index);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting brick-index-max to dict");
+                goto out;
+        }
+        ret = dict_set_int32 (rsp_dict, "other-count", other_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting other-count to dict");
+                goto out;
+        }
+        ret = dict_set_int32 (rsp_dict, "count", node_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting node count to dict");
+                goto out;
+        }
+
+        /* Active tasks */
+        /* Tasks are added only for normal volume status request for either a
+         * single volume or all volumes
+         */
+        if (!glusterd_status_has_tasks (cmd))
+                goto out;
+
+        ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
+        if (ret)
+                goto out;
+        ret = 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1391,7 +2658,7 @@ glusterd_op_ac_none (glusterd_op_sm_event_t *event, void *ctx)
 {
         int ret = 0;
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
@@ -1405,6 +2672,7 @@ glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
         xlator_t             *this     = NULL;
         glusterd_peerinfo_t  *peerinfo = NULL;
         uint32_t             pending_count = 0;
+        dict_t               *dict     = NULL;
 
         this = THIS;
         priv = this->private;
@@ -1419,21 +2687,61 @@ glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
                     (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
                         continue;
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_LOCK];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, peerinfo);
-                        if (ret)
-                                continue;
-                        pending_count++;
+                /* Based on the op_version, acquire a cluster or volume lock */
+                if (priv->op_version < GD_OP_VERSION_4) {
+                        proc = &peerinfo->mgmt->proctable
+                                          [GLUSTERD_MGMT_CLUSTER_LOCK];
+                        if (proc->fn) {
+                                ret = proc->fn (NULL, this, peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send lock request "
+                                                "for operation 'Volume %s' to "
+                                                "peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                } else {
+                        dict = glusterd_op_get_ctx ();
+                        dict_ref (dict);
+
+                        proc = &peerinfo->mgmt_v3->proctable
+                                          [GLUSTERD_MGMT_V3_VOLUME_LOCK];
+                        if (proc->fn) {
+                                ret = dict_set_static_ptr (dict, "peerinfo",
+                                                           peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set peerinfo");
+                                        dict_unref (dict);
+                                        goto out;
+                                }
+
+                                ret = proc->fn (NULL, this, dict);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send volume lock "
+                                                "request for operation "
+                                                "'Volume %s' to peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        dict_unref (dict);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
                 }
         }
 
         opinfo.pending_count = pending_count;
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
-
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
@@ -1446,17 +2754,12 @@ glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
         xlator_t             *this     = NULL;
         glusterd_peerinfo_t  *peerinfo = NULL;
         uint32_t             pending_count = 0;
+        dict_t               *dict     = NULL;
 
         this = THIS;
         priv = this->private;
         GF_ASSERT (priv);
 
-        /*ret = glusterd_unlock (priv->uuid);
-
-        if (ret)
-                goto out;
-        */
-
         list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
                 GF_ASSERT (peerinfo);
 
@@ -1466,23 +2769,63 @@ glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
                     (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
                         continue;
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_UNLOCK];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, peerinfo);
-                        if (ret)
-                                continue;
-                        pending_count++;
+                /* Based on the op_version,
+                 * release the cluster or volume lock */
+                if (priv->op_version < GD_OP_VERSION_4) {
+                        proc = &peerinfo->mgmt->proctable
+                                          [GLUSTERD_MGMT_CLUSTER_UNLOCK];
+                        if (proc->fn) {
+                                ret = proc->fn (NULL, this, peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send unlock request "
+                                                "for operation 'Volume %s' to "
+                                                "peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                } else {
+                        dict = glusterd_op_get_ctx ();
+                        dict_ref (dict);
+
+                        proc = &peerinfo->mgmt_v3->proctable
+                                          [GLUSTERD_MGMT_V3_VOLUME_UNLOCK];
+                        if (proc->fn) {
+                                ret = dict_set_static_ptr (dict, "peerinfo",
+                                                           peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set peerinfo");
+                                        dict_unref (dict);
+                                        goto out;
+                                }
+
+                                ret = proc->fn (NULL, this, dict);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send volume unlock "
+                                                "request for operation "
+                                                "'Volume %s' to peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        dict_unref (dict);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
                 }
         }
 
         opinfo.pending_count = pending_count;
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
-
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
-
 }
 
 static int
@@ -1494,9 +2837,10 @@ glusterd_op_ac_ack_drain (glusterd_op_sm_event_t *event, void *ctx)
                 opinfo.pending_count--;
 
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                                   &event->txn_id, NULL);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
@@ -1510,41 +2854,96 @@ glusterd_op_ac_send_unlock_drain (glusterd_op_sm_event_t *event, void *ctx)
 static int
 glusterd_op_ac_lock (glusterd_op_sm_event_t *event, void *ctx)
 {
-        glusterd_op_lock_ctx_t   *lock_ctx = NULL;
-        int32_t                  ret = 0;
-
+        int32_t                         ret             = 0;
+        char                           *volname         = NULL;
+        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
+        glusterd_conf_t                *priv            = NULL;
+        xlator_t                       *this            = NULL;
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
 
+        this = THIS;
+        priv = this->private;
+
         lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
 
-        ret = glusterd_lock (lock_ctx->uuid);
+        /* If the req came from a node running on older op_version
+         * the dict won't be present. Based on it acquiring a cluster
+         * or volume lock */
+        if (lock_ctx->dict == NULL) {
+                ret = glusterd_lock (lock_ctx->uuid);
+                glusterd_op_lock_send_resp (lock_ctx->req, ret);
+        } else {
+                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+                else {
+                        ret = glusterd_volume_lock (volname, lock_ctx->uuid);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to acquire lock for %s",
+                                        volname);
+                }
 
-        gf_log ("", GF_LOG_DEBUG, "Lock Returned %d", ret);
+                glusterd_op_volume_lock_send_resp (lock_ctx->req,
+                                                   &event->txn_id, ret);
 
-        glusterd_op_lock_send_resp (lock_ctx->req, ret);
+                dict_unref (lock_ctx->dict);
+        }
 
+        gf_log (THIS->name, GF_LOG_DEBUG, "Lock Returned %d", ret);
         return ret;
 }
 
 static int
 glusterd_op_ac_unlock (glusterd_op_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
-        glusterd_op_lock_ctx_t   *lock_ctx = NULL;
+        int32_t                         ret             = 0;
+        char                           *volname         = NULL;
+        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
+        glusterd_conf_t                *priv            = NULL;
+        xlator_t                       *this            = NULL;
+
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
 
+        this = THIS;
+        priv = this->private;
+
         lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
 
-        ret = glusterd_unlock (lock_ctx->uuid);
+        /* If the req came from a node running on older op_version
+         * the dict won't be present. Based on it releasing the cluster
+         * or volume lock */
+        if (lock_ctx->dict == NULL) {
+                ret = glusterd_unlock (lock_ctx->uuid);
+                glusterd_op_unlock_send_resp (lock_ctx->req, ret);
+        } else {
+                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+                else {
+                        ret = glusterd_volume_unlock (volname, lock_ctx->uuid);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                }
+
+                glusterd_op_volume_unlock_send_resp (lock_ctx->req,
+                                                     &event->txn_id, ret);
 
-        gf_log ("", GF_LOG_DEBUG, "Unlock Returned %d", ret);
+                dict_unref (lock_ctx->dict);
+        }
 
-        glusterd_op_unlock_send_resp (lock_ctx->req, ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
 
+        if (priv->pending_quorum_action)
+                glusterd_do_quorum_action ();
         return ret;
 }
 
@@ -1561,7 +2960,7 @@ glusterd_op_ac_local_unlock (glusterd_op_sm_event_t *event, void *ctx)
 
         ret = glusterd_unlock (*originator);
 
-        gf_log ("", GF_LOG_DEBUG, "Unlock Returned %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
 
         return ret;
 }
@@ -1579,74 +2978,243 @@ glusterd_op_ac_rcvd_lock_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
+                                           &event->txn_id, NULL);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+out:
+        return ret;
+}
 
+static int
+glusterd_dict_set_volid (dict_t *dict, char *volname, char **op_errstr)
+{
+        int                     ret = -1;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *volid = NULL;
+        char                    msg[1024] = {0,};
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dict || !volname)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                goto out;
+        }
+        volid = gf_strdup (uuid_utoa (volinfo->volume_id));
+        if (!volid) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_dynstr (dict, "vol-id", volid);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to set volume id of volume"
+                          " %s", volname);
+                goto out;
+        }
 out:
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
         return ret;
 }
 
 int
-glusterd_op_build_payload (dict_t **req)
+glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
 {
         int                     ret = -1;
         void                    *ctx = NULL;
+        dict_t                  *dict = NULL;
         dict_t                  *req_dict = NULL;
         glusterd_op_t           op = GD_OP_NONE;
+        char                    *volname = NULL;
+        uint32_t                status_cmd = GF_CLI_STATUS_NONE;
+        char                    *errstr = NULL;
+        xlator_t                *this = NULL;
 
         GF_ASSERT (req);
 
+        this = THIS;
+        GF_ASSERT (this);
+
         req_dict = dict_new ();
         if (!req_dict)
                 goto out;
 
-        op  = glusterd_op_get_op ();
-        ctx = (void*)glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR, "Null Context for "
-                        "op %d", op);
-                ret = -1;
-                goto out;
+        if (!op_ctx) {
+                op  = glusterd_op_get_op ();
+                ctx = (void*)glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log (this->name, GF_LOG_ERROR, "Null Context for "
+                                "op %d", op);
+                        ret = -1;
+                        goto out;
+                }
+
+        } else {
+#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
+                ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, (int32_t*)&op);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get volume"
+                                " operation");
+                        goto out;
+                }
+                ctx = op_ctx;
+#undef GD_SYNC_OPCODE_KEY
         }
 
+        dict = ctx;
         switch (op) {
                 case GD_OP_CREATE_VOLUME:
                         {
-                                dict_t  *dict = ctx;
                                 ++glusterfs_port;
-                                ret = dict_set_int32 (dict, "port", glusterfs_port);
-                                if (ret)
+                                ret = dict_set_int32 (dict, "port",
+                                                      glusterfs_port);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set port in "
+                                                "dictionary");
+                                        goto out;
+                                }
+                                dict_copy (dict, req_dict);
+                        }
+                        break;
+
+                case GD_OP_GSYNC_CREATE:
+                case GD_OP_GSYNC_SET:
+                        {
+                                ret = glusterd_op_gsync_args_get (dict,
+                                                                  &errstr,
+                                                                  &volname,
+                                                                  NULL, NULL);
+                                if (ret == 0) {
+                                        ret = glusterd_dict_set_volid
+                                                (dict, volname, op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
+                                dict_copy (dict, req_dict);
+                        }
+                        break;
+
+                case GD_OP_SET_VOLUME:
+                        {
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
                                         goto out;
+                                }
+                                if (strcmp (volname, "help") &&
+                                    strcmp (volname, "help-xml") &&
+                                    strcasecmp (volname, "all")) {
+                                        ret = glusterd_dict_set_volid
+                                                 (dict, volname, op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
+                                dict_destroy (req_dict);
+                                req_dict = dict_ref (dict);
+                        }
+                        break;
+
+                case GD_OP_SYNC_VOLUME:
+                        {
                                 dict_copy (dict, req_dict);
+                                break;
+                        }
+
+                case GD_OP_REMOVE_BRICK:
+                        {
+                                dict_t *dict = ctx;
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
+                                        goto out;
+                                }
+
+                                ret = glusterd_dict_set_volid (dict, volname,
+                                                               op_errstr);
+                                if (ret)
+                                        goto out;
+
+                                dict_destroy (req_dict);
+                                req_dict = dict_ref (dict);
                         }
                         break;
 
+                case GD_OP_STATUS_VOLUME:
+                        {
+                                ret = dict_get_uint32 (dict, "cmd",
+                                                       &status_cmd);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Status command not present "
+                                                "in op ctx");
+                                        goto out;
+                                }
+                                if (GF_CLI_STATUS_ALL & status_cmd) {
+                                        dict_copy (dict, req_dict);
+                                        break;
+                                }
+                        }
+                        /*fall-through*/
                 case GD_OP_DELETE_VOLUME:
                 case GD_OP_START_VOLUME:
                 case GD_OP_STOP_VOLUME:
                 case GD_OP_ADD_BRICK:
                 case GD_OP_REPLACE_BRICK:
-                case GD_OP_SET_VOLUME:
                 case GD_OP_RESET_VOLUME:
-                case GD_OP_REMOVE_BRICK:
                 case GD_OP_LOG_ROTATE:
-                case GD_OP_SYNC_VOLUME:
                 case GD_OP_QUOTA:
-                case GD_OP_GSYNC_SET:
                 case GD_OP_PROFILE_VOLUME:
-                case GD_OP_STATUS_VOLUME:
                 case GD_OP_REBALANCE:
                 case GD_OP_HEAL_VOLUME:
                 case GD_OP_STATEDUMP_VOLUME:
                 case GD_OP_CLEARLOCKS_VOLUME:
                 case GD_OP_DEFRAG_BRICK_VOLUME:
                         {
-                                dict_t  *dict = ctx;
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
+                                        goto out;
+                                }
+
+                                if (strcasecmp (volname, "all")) {
+                                        ret = glusterd_dict_set_volid (dict,
+                                                                       volname,
+                                                                     op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
                                 dict_copy (dict, req_dict);
                         }
                         break;
 
+                case GD_OP_COPY_FILE:
+                        {
+                                dict_copy (dict, req_dict);
+                                break;
+                        }
+
+                case GD_OP_SYS_EXEC:
+                        {
+                                dict_copy (dict, req_dict);
+                                break;
+                        }
+
                 default:
                         break;
         }
@@ -1658,6 +3226,105 @@ out:
         return ret;
 }
 
+gf_boolean_t
+glusterd_is_get_op (xlator_t *this, glusterd_op_t op, dict_t *dict)
+{
+        char            *key = NULL;
+        char            *volname = NULL;
+        int             ret = 0;
+
+        if (op == GD_OP_STATUS_VOLUME)
+                return _gf_true;
+
+        if ((op == GD_OP_SET_VOLUME)) {
+                //check for set volume help
+                ret = dict_get_str (dict, "volname", &volname);
+                if (volname &&
+                    ((strcmp (volname, "help") == 0) ||
+                     (strcmp (volname, "help-xml") == 0))) {
+                        ret = dict_get_str (dict, "key1", &key);
+                        if (ret < 0)
+                                return _gf_true;
+                }
+        }
+
+        return _gf_false;
+}
+
+gf_boolean_t
+glusterd_is_op_quorum_validation_required (xlator_t *this, glusterd_op_t op,
+                                           dict_t *dict)
+{
+        gf_boolean_t    required = _gf_true;
+        char            *key = NULL;
+        char            *key_fixed = NULL;
+        int             ret = -1;
+
+        if (glusterd_is_get_op (this, op, dict)) {
+                required = _gf_false;
+                goto out;
+        }
+        if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
+                goto out;
+        if (op == GD_OP_SET_VOLUME)
+                ret = dict_get_str (dict, "key1", &key);
+        else if (op == GD_OP_RESET_VOLUME)
+                ret = dict_get_str (dict, "key", &key);
+        if (ret)
+                goto out;
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0)
+                goto out;
+        if (key_fixed)
+                key = key_fixed;
+        if (glusterd_is_quorum_option (key))
+                required = _gf_false;
+out:
+        GF_FREE (key_fixed);
+        return required;
+}
+
+static int
+glusterd_op_validate_quorum (xlator_t *this, glusterd_op_t op,
+                             dict_t *dict, char **op_errstr)
+{
+        int                     ret = 0;
+        char                    *volname = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *errstr = NULL;
+
+
+        errstr = "Quorum not met. Volume operation not allowed.";
+        if (!glusterd_is_op_quorum_validation_required (this, op, dict))
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        if (does_gd_meet_server_quorum (this)) {
+                ret = 0;
+                goto out;
+        }
+
+        if (glusterd_is_volume_in_server_quorum (volinfo)) {
+                ret = -1;
+                *op_errstr = gf_strdup (errstr);
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
 static int
 glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 {
@@ -1678,14 +3345,32 @@ glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 
         op = glusterd_op_get_op ();
 
-        ret = glusterd_op_build_payload (&dict);
-        if (ret)
+        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                opinfo.op_errstr = op_errstr;
                 goto out;
+        }
+
+        ret = glusterd_op_validate_quorum (this, op, dict, &op_errstr);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
 
         /* rsp_dict NULL from source */
         ret = glusterd_op_stage_validate (op, dict, &op_errstr, NULL);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Staging failed");
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
+                        gd_op_list[op], "localhost",
+                        (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_STAGE_FAIL,
+                                     "localhost");
                 opinfo.op_errstr = op_errstr;
                 goto out;
         }
@@ -1704,13 +3389,19 @@ glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
                 if (proc->fn) {
                         ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
+                                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                        "set peerinfo");
                                 goto out;
                         }
 
                         ret = proc->fn (NULL, this, dict);
-                        if (ret)
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                        "send stage request for operation "
+                                        "'Volume %s' to peer %s",
+                                        gd_op_list[op], peerinfo->hostname);
                                 continue;
+                        }
                         pending_count++;
                 }
         }
@@ -1720,27 +3411,29 @@ out:
         if (dict)
                 dict_unref (dict);
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             &event->txn_id, NULL);
                 opinfo.op_ret = ret;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Sent op req to %d peers",
+        gf_log (this->name, GF_LOG_DEBUG, "Sent stage op request for "
+                "'Volume %s' to %d peers", gd_op_list[op],
                 opinfo.pending_count);
 
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 
 }
 
 static int32_t
-glusterd_op_start_rb_timer (dict_t *dict)
+glusterd_op_start_rb_timer (dict_t *dict, uuid_t *txn_id)
 {
         int32_t         op = 0;
-        struct timeval  timeout = {0, };
+        struct timespec timeout = {0, };
         glusterd_conf_t *priv = NULL;
         int32_t         ret = -1;
         dict_t          *rb_ctx = NULL;
@@ -1755,13 +3448,13 @@ glusterd_op_start_rb_timer (dict_t *dict)
                 goto out;
         }
 
-        if (op == GF_REPLACE_OP_START ||
-            op == GF_REPLACE_OP_ABORT)
-                timeout.tv_sec  = 5;
-        else
-                timeout.tv_sec = 1;
+        if (op != GF_REPLACE_OP_START) {
+                ret = glusterd_op_sm_inject_all_acc (txn_id);
+                goto out;
+        }
 
-        timeout.tv_usec = 0;
+        timeout.tv_sec  = 5;
+        timeout.tv_nsec = 0;
 
 
         rb_ctx = dict_copy (dict, rb_ctx);
@@ -1771,6 +3464,17 @@ glusterd_op_start_rb_timer (dict_t *dict)
                 ret = -1;
                 goto out;
         }
+
+        ret = dict_set_bin (rb_ctx, "transaction_id",
+                            txn_id, sizeof (uuid_t));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
+        } else
+               gf_log ("", GF_LOG_DEBUG,
+                        "transaction_id = %s", uuid_utoa (*txn_id));
+
         priv->timer = gf_timer_call_after (THIS->ctx, timeout,
                                            glusterd_do_replace_brick,
                                            (void *) rb_ctx);
@@ -1781,6 +3485,403 @@ out:
         return ret;
 }
 
+/* This function takes a dict and converts the uuid values of key specified
+ * into hostnames
+ */
+static int
+glusterd_op_volume_dict_uuid_to_hostname (dict_t *dict, const char *key_fmt,
+                                          int idx_min, int idx_max)
+{
+        int             ret = -1;
+        int             i = 0;
+        char            key[1024];
+        char            *uuid_str = NULL;
+        uuid_t          uuid = {0,};
+        char            *hostname = NULL;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key_fmt);
+
+        for (i = idx_min; i < idx_max; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), key_fmt, i);
+                ret = dict_get_str (dict, key, &uuid_str);
+                if (ret)
+                        continue;
+
+                gf_log (this->name, GF_LOG_DEBUG, "Got uuid %s",
+                        uuid_str);
+
+                ret = uuid_parse (uuid_str, uuid);
+                /* if parsing fails don't error out
+                 * let the original value be retained
+                 */
+                if (ret)
+                        continue;
+
+                hostname = glusterd_uuid_to_hostname (uuid);
+                if (hostname) {
+                        gf_log (this->name, GF_LOG_DEBUG, "%s -> %s",
+                                uuid_str, hostname);
+                        ret = dict_set_dynstr (dict, key, hostname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Error setting hostname %s to dict",
+                                        hostname);
+                                GF_FREE (hostname);
+                                goto out;
+                        }
+                }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+reassign_defrag_status (dict_t *dict, char *key, gf_defrag_status_t *status)
+{
+        int ret = 0;
+
+        if (!*status)
+                return ret;
+
+        switch (*status) {
+        case GF_DEFRAG_STATUS_STARTED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
+                break;
+
+        case GF_DEFRAG_STATUS_STOPPED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED;
+                break;
+
+        case GF_DEFRAG_STATUS_COMPLETE:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
+                break;
+
+        case GF_DEFRAG_STATUS_FAILED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED;
+                break;
+        default:
+                break;
+         }
+
+        ret = dict_set_int32(dict, key, *status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to reset defrag %s in dict", key);
+
+        return ret;
+}
+
+/* Check and reassign the defrag_status enum got from the rebalance process
+ * of all peers so that the rebalance-status CLI command can display if a
+ * full-rebalance or just a fix-layout was carried out.
+ */
+static int
+glusterd_op_check_peer_defrag_status (dict_t *dict, int count)
+{
+        glusterd_volinfo_t *volinfo  = NULL;
+        gf_defrag_status_t status    = GF_DEFRAG_STATUS_NOT_STARTED;
+        char               key[256]  = {0,};
+        char               *volname  = NULL;
+        int                ret       = -1;
+        int                i         = 1;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        if (volinfo->rebal.defrag_cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                /* Fix layout was not issued; we don't need to reassign
+                   the status */
+                ret = 0;
+                goto out;
+        }
+
+        do {
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to get defrag %s", key);
+                        goto out;
+                }
+                ret = reassign_defrag_status (dict, key, &status);
+                if (ret)
+                        goto out;
+                i++;
+        } while (i <= count);
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+/* This function is used to modify the op_ctx dict before sending it back
+ * to cli. This is useful in situations like changing the peer uuids to
+ * hostnames etc.
+ */
+void
+glusterd_op_modify_op_ctx (glusterd_op_t op, void *ctx)
+{
+        int             ret = -1;
+        dict_t          *op_ctx = NULL;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             count = 0;
+        uint32_t        cmd = GF_CLI_STATUS_NONE;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (ctx)
+                op_ctx = ctx;
+        else
+                op_ctx = glusterd_op_get_ctx();
+
+        if (!op_ctx) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Operation context is not present.");
+                goto out;
+        }
+
+        switch (op) {
+        case GD_OP_STATUS_VOLUME:
+                ret = dict_get_uint32 (op_ctx, "cmd", &cmd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get status cmd");
+                        goto out;
+                }
+                if (!(cmd & GF_CLI_STATUS_NFS || cmd & GF_CLI_STATUS_SHD ||
+                    (cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "op_ctx modification not required for status "
+                                "operation being performed");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (op_ctx, "brick-index-max",
+                                      &brick_index_max);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get brick-index-max");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (op_ctx, "other-count", &other_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get other-count");
+                        goto out;
+                }
+
+                count = brick_index_max + other_count + 1;
+
+                /* add 'brick%d.peerid' into op_ctx with value of 'brick%d.path'.
+                   nfs/sshd like services have this additional uuid */
+                {
+                        char  key[1024];
+                        char *uuid_str = NULL;
+                        char *uuid = NULL;
+                        int   i;
+
+                        for (i = brick_index_max + 1; i < count; i++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "brick%d.path", i);
+                                ret = dict_get_str (op_ctx, key, &uuid_str);
+                                if (!ret) {
+                                        memset (key, 0, sizeof (key));
+                                        snprintf (key, sizeof (key),
+                                                  "brick%d.peerid", i);
+                                        uuid = gf_strdup (uuid_str);
+                                        if (!uuid) {
+                                                gf_log (this->name, GF_LOG_DEBUG,
+                                                        "unable to create dup of"
+                                                        " uuid_str");
+                                                continue;
+                                        }
+                                        ret = dict_set_dynstr (op_ctx, key,
+                                                               uuid);
+                                        if (ret != 0) {
+                                                GF_FREE (uuid);
+                                        }
+                                }
+                        }
+                }
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "brick%d.path",
+                                                                0, count);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                break;
+
+        case GD_OP_PROFILE_VOLUME:
+                ret = dict_get_str_boolean (op_ctx, "nfs", _gf_false);
+                if (!ret)
+                        goto out;
+
+                ret = dict_get_int32 (op_ctx, "count", &count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get brick count");
+                        goto out;
+                }
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "%d-brick",
+                                                                1, (count + 1));
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                break;
+
+        /* For both rebalance and remove-brick status, the glusterd op is the
+         * same
+         */
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = dict_get_int32 (op_ctx, "count", &count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get count");
+                        goto out;
+                }
+
+                /* add 'node-name-%d' into op_ctx with value uuid_str.
+                   this will be used to convert to hostname later */
+                {
+                        char  key[1024];
+                        char *uuid_str = NULL;
+                        char *uuid = NULL;
+                        int   i;
+
+                        for (i = 1; i <= count; i++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "node-uuid-%d", i);
+                                ret = dict_get_str (op_ctx, key, &uuid_str);
+                                if (!ret) {
+                                        memset (key, 0, sizeof (key));
+                                        snprintf (key, sizeof (key),
+                                                  "node-name-%d", i);
+                                        uuid = gf_strdup (uuid_str);
+                                        if (!uuid) {
+                                                gf_log (this->name, GF_LOG_DEBUG,
+                                                        "unable to create dup of"
+                                                        " uuid_str");
+                                                continue;
+                                        }
+                                        ret = dict_set_dynstr (op_ctx, key,
+                                                               uuid);
+                                        if (ret != 0) {
+                                                GF_FREE (uuid);
+                                        }
+                                }
+                        }
+                }
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "node-name-%d",
+                                                                1, (count + 1));
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                ret = glusterd_op_check_peer_defrag_status (op_ctx, count);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to reset defrag status for fix-layout");
+                break;
+
+        default:
+                ret = 0;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "op_ctx modification not required");
+                break;
+
+        }
+
+out:
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "op_ctx modification failed");
+        return;
+}
+
+static int
+glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_commit_hook_type_t type)
+{
+        glusterd_conf_t *priv                   = NULL;
+        char            hookdir[PATH_MAX]       = {0, };
+        char            scriptdir[PATH_MAX]     = {0, };
+        char            type_subdir[256]        = {0, };
+        char            *cmd_subdir             = NULL;
+        int             ret                     = -1;
+
+        priv = THIS->private;
+        switch (type) {
+                case GD_COMMIT_HOOK_NONE:
+                case GD_COMMIT_HOOK_MAX:
+                        /*Won't be called*/
+                        break;
+
+                case GD_COMMIT_HOOK_PRE:
+                        strcpy (type_subdir, "pre");
+                        break;
+                case GD_COMMIT_HOOK_POST:
+                        strcpy (type_subdir, "post");
+                        break;
+        }
+
+        cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
+        if (strlen (cmd_subdir) == 0)
+                return -1;
+
+        GLUSTERD_GET_HOOKS_DIR (hookdir, GLUSTERD_HOOK_VER, priv);
+        snprintf (scriptdir, sizeof (scriptdir), "%s/%s/%s",
+                  hookdir, cmd_subdir, type_subdir);
+
+        switch (type) {
+                case GD_COMMIT_HOOK_NONE:
+                case GD_COMMIT_HOOK_MAX:
+                        /*Won't be called*/
+                        break;
+
+                case GD_COMMIT_HOOK_PRE:
+                        ret = glusterd_hooks_run_hooks (scriptdir, op, op_ctx,
+                                                        type);
+                        break;
+                case GD_COMMIT_HOOK_POST:
+                        ret = glusterd_hooks_post_stub_enqueue (scriptdir, op,
+                                                                op_ctx);
+                        break;
+        }
+
+        return ret;
+}
+
 static int
 glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
 {
@@ -1800,19 +3901,32 @@ glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
         priv = this->private;
         GF_ASSERT (priv);
 
-        op = glusterd_op_get_op ();
-        ret = glusterd_op_build_payload (&dict);
+        op      = glusterd_op_get_op ();
+        op_dict = glusterd_op_get_ctx ();
 
-        if (ret)
+        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                opinfo.op_errstr = op_errstr;
                 goto out;
+        }
 
         ret = glusterd_op_commit_perform (op, dict, &op_errstr, NULL); //rsp_dict invalid for source
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Commit failed");
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
+                        gd_op_list[op], "localhost", (op_errstr) ? ":" : " ",
+                        (op_errstr) ? op_errstr : " ");
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_COMMIT_FAIL,
+                                     "localhost");
                 opinfo.op_errstr = op_errstr;
                 goto out;
         }
 
+
         list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
                 GF_ASSERT (peerinfo);
 
@@ -1827,41 +3941,48 @@ glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
                 if (proc->fn) {
                         ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to set peerinfo");
                                 goto out;
                         }
                         ret = proc->fn (NULL, this, dict);
-                        if (ret)
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                        "send commit request for operation "
+                                        "'Volume %s' to peer %s",
+                                        gd_op_list[op], peerinfo->hostname);
                                 continue;
+                        }
                         pending_count++;
                 }
         }
 
         opinfo.pending_count = pending_count;
-        gf_log ("glusterd", GF_LOG_INFO, "Sent op req to %d peers",
-                opinfo.pending_count);
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], opinfo.pending_count);
 out:
         if (dict)
                 dict_unref (dict);
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             &event->txn_id, NULL);
                 opinfo.op_ret = ret;
         }
 
         if (!opinfo.pending_count) {
                 if (op == GD_OP_REPLACE_BRICK) {
-                        op_dict = glusterd_op_get_ctx ();
-                        ret = glusterd_op_start_rb_timer (op_dict);
+                        ret = glusterd_op_start_rb_timer (op_dict,
+                                                          &event->txn_id);
 
                 } else {
-
-                        ret = glusterd_op_sm_inject_all_acc ();
+                        glusterd_op_modify_op_ctx (op, NULL);
+                        ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
                 }
                 goto err;
         }
 
 err:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 
@@ -1880,10 +4001,11 @@ glusterd_op_ac_rcvd_stage_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC,
+                                           &event->txn_id, NULL);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1901,10 +4023,11 @@ glusterd_op_ac_stage_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, NULL);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1922,10 +4045,11 @@ glusterd_op_ac_commit_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, NULL);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1936,6 +4060,10 @@ glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         int                     ret = 0;
         glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
         gf_boolean_t                free_errstr = _gf_false;
+        xlator_t                    *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
@@ -1943,7 +4071,7 @@ glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
 
         ret = glusterd_remove_pending_entry (&opinfo.pending_bricks, ev_ctx->pending_node->node);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "unknown response received ");
+                gf_log (this->name, GF_LOG_ERROR, "unknown response received ");
                 ret = -1;
                 free_errstr = _gf_true;
                 goto out;
@@ -1961,7 +4089,8 @@ glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.brick_pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, ev_ctx->commit_ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, ev_ctx->commit_ctx);
 
 out:
         if (ev_ctx->rsp_dict)
@@ -1969,7 +4098,7 @@ out:
         if (free_errstr && ev_ctx->op_errstr)
                 GF_FREE (ev_ctx->op_errstr);
         GF_FREE (ctx);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1977,11 +4106,14 @@ out:
 static int
 glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
 {
-        dict_t                 *dict              = NULL;
+        dict_t                 *op_ctx            = NULL;
         int                     ret               = 0;
         gf_boolean_t            commit_ack_inject = _gf_true;
         glusterd_op_t           op                = GD_OP_NONE;
+        xlator_t               *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         op = glusterd_op_get_op ();
         GF_ASSERT (event);
 
@@ -1992,17 +4124,17 @@ glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
                 goto out;
 
         if (op == GD_OP_REPLACE_BRICK) {
-                dict = glusterd_op_get_ctx ();
-                if (!dict) {
-                        gf_log (THIS->name, GF_LOG_CRITICAL, "Operation "
+                op_ctx = glusterd_op_get_ctx ();
+                if (!op_ctx) {
+                        gf_log (this->name, GF_LOG_CRITICAL, "Operation "
                                 "context is not present.");
                         ret = -1;
                         goto out;
                 }
 
-                ret = glusterd_op_start_rb_timer (dict);
+                ret = glusterd_op_start_rb_timer (op_ctx, &event->txn_id);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Couldn't start "
+                        gf_log (this->name, GF_LOG_ERROR, "Couldn't start "
                                 "replace-brick operation.");
                         goto out;
                 }
@@ -2011,13 +4143,19 @@ glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
                 goto out;
         }
 
+
 out:
         if (commit_ack_inject) {
                 if (ret)
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
-                else if (!opinfo.pending_count)
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC, NULL);
-
+                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                                           &event->txn_id,
+                                                           NULL);
+                else if (!opinfo.pending_count) {
+                        glusterd_op_modify_op_ctx (op, NULL);
+                        ret = glusterd_op_sm_inject_event
+                                                  (GD_OP_EVENT_COMMIT_ACC,
+                                                   &event->txn_id, NULL);
+                }
                 /*else do nothing*/
         }
 
@@ -2037,9 +4175,10 @@ glusterd_op_ac_rcvd_unlock_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
+                                           &event->txn_id, NULL);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
 out:
         return ret;
@@ -2071,7 +4210,7 @@ glusterd_op_reset_ctx ()
 }
 
 int32_t
-glusterd_op_txn_complete ()
+glusterd_op_txn_complete (uuid_t *txn_id)
 {
         int32_t                 ret = -1;
         glusterd_conf_t         *priv = NULL;
@@ -2081,9 +4220,13 @@ glusterd_op_txn_complete ()
         rpcsvc_request_t        *req = NULL;
         void                    *ctx = NULL;
         char                    *op_errstr = NULL;
+        char                    *volname = NULL;
+        xlator_t                *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
 
-        priv = THIS->private;
+        priv = this->private;
         GF_ASSERT (priv);
 
         op  = glusterd_op_get_op ();
@@ -2100,32 +4243,55 @@ glusterd_op_txn_complete ()
         glusterd_op_reset_ctx ();
         glusterd_op_clear_errstr ();
 
-        ret = glusterd_unlock (priv->uuid);
-
-        /* unlock cant/shouldnt fail here!! */
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to clear local lock, ret: %d", ret);
+        /* Based on the op-version, we release the cluster or volume lock */
+        if (priv->op_version < GD_OP_VERSION_4) {
+                ret = glusterd_unlock (MY_UUID);
+                /* unlock cant/shouldnt fail here!! */
+                if (ret)
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to clear local lock, ret: %d", ret);
+                else
+                        gf_log (this->name, GF_LOG_DEBUG, "Cleared local lock");
         } else {
-                gf_log ("glusterd", GF_LOG_INFO, "Cleared local lock");
+                ret = dict_get_str (ctx, "volname", &volname);
+                if (ret)
+                        gf_log ("", GF_LOG_INFO,
+                                "No Volume name present. "
+                                "Locks have not been held.");
+
+                if (volname) {
+                        ret = glusterd_volume_unlock (volname, MY_UUID);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                }
         }
 
         ret = glusterd_op_send_cli_response (op, op_ret,
                                              op_errno, req, ctx, op_errstr);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Responding to cli failed, ret: %d",
-                        ret);
+                gf_log (this->name, GF_LOG_ERROR, "Responding to cli failed, "
+                        "ret: %d", ret);
                 //Ignore this error, else state machine blocks
                 ret = 0;
         }
 
-        glusterd_op_free_ctx (op, ctx);
         if (op_errstr && (strcmp (op_errstr, "")))
                 GF_FREE (op_errstr);
 
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        if (priv->pending_quorum_action)
+                glusterd_do_quorum_action ();
+
+        /* Clearing the transaction opinfo */
+        ret = glusterd_clear_txn_opinfo (txn_id);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to clear transaction's opinfo");
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -2136,9 +4302,9 @@ glusterd_op_ac_unlocked_all (glusterd_op_sm_event_t *event, void *ctx)
 
         GF_ASSERT (event);
 
-        ret = glusterd_op_txn_complete ();
+        ret = glusterd_op_txn_complete (&event->txn_id);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -2152,7 +4318,11 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
         dict_t                  *rsp_dict  = NULL;
         char                    *op_errstr = NULL;
         dict_t                  *dict = NULL;
+        xlator_t                *this = NULL;
+        uuid_t                  *txn_id = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (ctx);
 
         req_ctx = ctx;
@@ -2161,8 +4331,8 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 
         rsp_dict = dict_new ();
         if (!rsp_dict) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Out of memory");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
                 return -1;
         }
 
@@ -2170,16 +4340,36 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
                                              rsp_dict);
 
         if (status) {
-                gf_log ("", GF_LOG_ERROR, "Validate failed: %d", status);
+                gf_log (this->name, GF_LOG_ERROR, "Stage failed on operation"
+                        " 'Volume %s', Status : %d", gd_op_list[req_ctx->op],
+                        status);
+        }
+
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+
+        if (txn_id)
+                uuid_copy (*txn_id, event->txn_id);
+        else {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_bin (rsp_dict, "transaction_id",
+                            txn_id, sizeof(uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
         }
 
         ret = glusterd_op_stage_send_resp (req_ctx->req, req_ctx->op,
                                            status, op_errstr, rsp_dict);
 
+out:
         if (op_errstr && (strcmp (op_errstr, "")))
                 GF_FREE (op_errstr);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         if (rsp_dict)
                 dict_unref (rsp_dict);
@@ -2208,7 +4398,7 @@ glusterd_need_brick_op (glusterd_op_t op)
         return ret;
 }
 
-static dict_t*
+dict_t*
 glusterd_op_init_commit_rsp_dict (glusterd_op_t op)
 {
         dict_t                  *rsp_dict = NULL;
@@ -2231,12 +4421,16 @@ static int
 glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
 {
         int                       ret        = 0;
-        glusterd_req_ctx_t       *req_ctx = NULL;
+        glusterd_req_ctx_t       *req_ctx    = NULL;
         int32_t                   status     = 0;
         char                     *op_errstr  = NULL;
         dict_t                   *dict       = NULL;
-        dict_t                   *rsp_dict = NULL;
+        dict_t                   *rsp_dict   = NULL;
+        xlator_t                 *this       = NULL;
+        uuid_t                   *txn_id     = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (ctx);
 
         req_ctx = ctx;
@@ -2247,6 +4441,7 @@ glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
         if (NULL == rsp_dict)
                 return -1;
 
+
         if (GD_OP_CLEARLOCKS_VOLUME == req_ctx->op) {
                 /*clear locks should be run only on
                  * originator glusterd*/
@@ -2257,21 +4452,39 @@ glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
                                                      &op_errstr, rsp_dict);
         }
 
-        if (status) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Commit failed: %d", status);
+        if (status)
+                gf_log (this->name, GF_LOG_ERROR, "Commit of operation "
+                        "'Volume %s' failed: %d", gd_op_list[req_ctx->op],
+                        status);
+
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+
+        if (txn_id)
+                uuid_copy (*txn_id, event->txn_id);
+        else {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_bin (rsp_dict, "transaction_id",
+                            txn_id, sizeof(uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
         }
 
         ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
                                             status, op_errstr, rsp_dict);
 
-        glusterd_op_fini_ctx ();
+out:
         if (op_errstr && (strcmp (op_errstr, "")))
                 GF_FREE (op_errstr);
 
         if (rsp_dict)
                 dict_unref (rsp_dict);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
@@ -2293,13 +4506,12 @@ glusterd_op_ac_send_commit_failed (glusterd_op_sm_event_t *event, void *ctx)
                                             opinfo.op_ret, opinfo.op_errstr,
                                             op_ctx);
 
-        glusterd_op_fini_ctx ();
         if (opinfo.op_errstr && (strcmp (opinfo.op_errstr, ""))) {
                 GF_FREE (opinfo.op_errstr);
                 opinfo.op_errstr = NULL;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
@@ -2330,6 +4542,7 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                             dict_t *rsp_dict)
 {
         int ret = -1;
+        xlator_t *this = THIS;
 
         switch (op) {
                 case GD_OP_CREATE_VOLUME:
@@ -2377,6 +4590,10 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_stage_sync_volume (dict, op_errstr);
                         break;
 
+                case GD_OP_GSYNC_CREATE:
+                        ret = glusterd_op_stage_gsync_create (dict, op_errstr);
+                        break;
+
                 case GD_OP_GSYNC_SET:
                         ret = glusterd_op_stage_gsync_set (dict, op_errstr);
                         break;
@@ -2386,7 +4603,8 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         break;
 
                 case GD_OP_QUOTA:
-                        ret = glusterd_op_stage_quota (dict, op_errstr);
+                        ret = glusterd_op_stage_quota (dict, op_errstr,
+                                                       rsp_dict);
                         break;
 
                 case GD_OP_STATUS_VOLUME:
@@ -2411,13 +4629,20 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                                                                    op_errstr);
                         break;
 
+                case GD_OP_COPY_FILE:
+                        ret = glusterd_op_stage_copy_file (dict, op_errstr);
+                        break;
+
+                case GD_OP_SYS_EXEC:
+                        ret = glusterd_op_stage_sys_exec (dict, op_errstr);
+                        break;
+
                 default:
-                        gf_log ("", GF_LOG_ERROR, "Unknown op %d",
-                                op);
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown op %s",
+                                gd_op_list[op]);
         }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
         return ret;
 }
 
@@ -2427,7 +4652,9 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                             dict_t *rsp_dict)
 {
         int ret = -1;
+        xlator_t *this = THIS;
 
+        glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_PRE);
         switch (op) {
                 case GD_OP_CREATE_VOLUME:
                         ret = glusterd_op_create_volume (dict, op_errstr);
@@ -2458,7 +4685,7 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         break;
 
                 case GD_OP_RESET_VOLUME:
-                        ret = glusterd_op_reset_volume (dict);
+                        ret = glusterd_op_reset_volume (dict, op_errstr);
                         break;
 
                 case GD_OP_REMOVE_BRICK:
@@ -2473,6 +4700,11 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_sync_volume (dict, op_errstr, rsp_dict);
                         break;
 
+                case GD_OP_GSYNC_CREATE:
+                        ret = glusterd_op_gsync_create (dict, op_errstr,
+                                                        rsp_dict);
+                        break;
+
                 case GD_OP_GSYNC_SET:
                         ret = glusterd_op_gsync_set (dict, op_errstr, rsp_dict);
                         break;
@@ -2483,7 +4715,7 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         break;
 
                 case GD_OP_QUOTA:
-                        ret = glusterd_op_quota (dict, op_errstr);
+                        ret = glusterd_op_quota (dict, op_errstr, rsp_dict);
                         break;
 
                 case GD_OP_STATUS_VOLUME:
@@ -2504,329 +4736,35 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         break;
 
                 case GD_OP_CLEARLOCKS_VOLUME:
-                        ret = glusterd_op_clearlocks_volume (dict, op_errstr);
+                        ret = glusterd_op_clearlocks_volume (dict, op_errstr,
+                                                             rsp_dict);
                         break;
 
-                default:
-                        gf_log ("", GF_LOG_ERROR, "Unknown op %d",
-                                op);
+                case GD_OP_COPY_FILE:
+                        ret = glusterd_op_copy_file (dict, op_errstr);
                         break;
-        }
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-void
-_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
-                             void *data)
-{
-        char    new_key[256] = {0};
-        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
-        data_t  *new_value = NULL;
-
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
-        snprintf (new_key, sizeof (new_key), "%d-%s", rsp_ctx->count, key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
-}
-
-int
-glusterd_profile_volume_brick_rsp (glusterd_brickinfo_t *brickinfo,
-                                   dict_t *rsp_dict, dict_t *op_ctx,
-                                   char **op_errstr)
-{
-        int     ret = 0;
-        glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
-        int32_t count = 0;
-        char    brick[PATH_MAX+1024] = {0};
-        char    key[256] = {0};
-        char    *full_brick = NULL;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (brickinfo);
-
-        ret = dict_get_int32 (op_ctx, "count", &count);
-        if (ret) {
-                count = 1;
-        } else {
-                count++;
-        }
-        snprintf (key, sizeof (key), "%d-brick", count);
-        snprintf (brick, sizeof (brick), "%s:%s", brickinfo->hostname,
-                  brickinfo->path);
-        full_brick = gf_strdup (brick);
-        GF_ASSERT (full_brick);
-        ret = dict_set_dynstr (op_ctx, key, full_brick);
-
-        rsp_ctx.count = count;
-        rsp_ctx.dict = op_ctx;
-        dict_foreach (rsp_dict, _profile_volume_add_brick_rsp, &rsp_ctx);
-        dict_del (op_ctx, "count");
-        ret = dict_set_int32 (op_ctx, "count", count);
-        return ret;
-}
-
-void
-_status_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
-                              void *data)
-{
-        char                            new_key[256] = {0,};
-        data_t                          *new_value = 0;
-        glusterd_pr_brick_rsp_conv_t    *rsp_ctx = NULL;
-
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        snprintf (new_key, sizeof (new_key), "brick%d.%s", rsp_ctx->count, key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
-
-        return;
-}
-
-//input-key: <replica-id>:<child-id>-*
-//output-key: <brick-id>-*
-void
-_heal_volume_add_shd_rsp (dict_t *this, char *key, data_t *value, void *data)
-{
-        char                            new_key[256] = {0,};
-        char                            int_str[16] = {0};
-        data_t                          *new_value = NULL;
-        char                            *rxl_end = NULL;
-        char                            *rxl_child_end = NULL;
-        glusterd_volinfo_t              *volinfo = NULL;
-        int                             rxl_id = 0;
-        int                             rxl_child_id = 0;
-        int                             brick_id = 0;
-        int                             int_len = 0;
-        int                             brick_count = 0;
-        int                             ret = 0;
-        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
-
-        rsp_ctx = data;
-        rxl_end = strchr (key, '-');
-        if (!rxl_end)
-                goto out;
-
-        int_len = strlen (key) - strlen (rxl_end);
-        strncpy (int_str, key, int_len);
-        int_str[int_len] = '\0';
-        ret = gf_string2int (int_str, &rxl_id);
-        if (ret)
-                goto out;
-
-        rxl_child_end = strchr (rxl_end + 1, '-');
-        if (!rxl_child_end)
-                goto out;
-
-        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
-        strncpy (int_str, rxl_end + 1, int_len);
-        int_str[int_len] = '\0';
-        ret = gf_string2int (int_str, &rxl_child_id);
-        if (ret)
-                goto out;
-
-        volinfo = rsp_ctx->volinfo;
-        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
-
-        new_value = data_copy (value);
-        snprintf (new_key, sizeof (new_key), "%d%s", brick_id, rxl_child_end);
-        dict_set (rsp_ctx->dict, new_key, new_value);
-
-        ret = dict_get_int32 (rsp_ctx->dict, "count", &brick_count);
-        if (brick_id >= brick_count)
-                ret = dict_set_int32 (rsp_ctx->dict, "count", brick_id + 1);
-out:
-        return;
-}
-
-int
-glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
-                                dict_t *op_ctx, char **op_errstr)
-{
-        int                             ret = 0;
-        glusterd_heal_rsp_conv_t        rsp_ctx = {0};
-        char                            *volname = NULL;
-        glusterd_volinfo_t              *volinfo = NULL;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (op_errstr);
-
-        ret = dict_get_str (req_dict, "volname", &volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
-                goto out;
-
-        rsp_ctx.dict = op_ctx;
-        rsp_ctx.volinfo = volinfo;
-        dict_foreach (rsp_dict, _heal_volume_add_shd_rsp, &rsp_ctx);
-
-out:
-        return ret;
-}
-
-int
-glusterd_status_volume_brick_rsp (glusterd_brickinfo_t *brickinfo,
-                                  dict_t *rsp_dict, dict_t *op_ctx,
-                                  char **op_errstr)
-{
-        int                             ret = 0;
-        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
-        int32_t                         count = 0;
-        int                             index = 0;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (brickinfo);
-
-        ret = dict_get_int32 (op_ctx, "count", &count);
-        if (ret) {
-                count = 0;
-        } else {
-                count++;
-        }
-        ret = dict_get_int32 (rsp_dict, "index", &index);
-        if (ret)
-                goto out;
-        dict_del (rsp_dict, "index");
-
-        rsp_ctx.count = index;
-        rsp_ctx.dict = op_ctx;
-        dict_foreach (rsp_dict, _status_volume_add_brick_rsp, &rsp_ctx);
-
-out:
-        return ret;
-}
-
-
-int
-glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
-                                 dict_t *op_ctx)
-{
-        int                             ret = 0;
-        char                            *volname = NULL;
-        glusterd_volinfo_t              *volinfo = NULL;
-        uint64_t                        files = 0;
-        uint64_t                        size = 0;
-        uint64_t                        lookup = 0;
-        gf_defrag_status_t              status = GF_DEFRAG_STATUS_NOT_STARTED;
-
-        GF_ASSERT (req_dict);
-
-        ret = dict_get_str (req_dict, "volname", &volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret  = glusterd_volinfo_find (volname, &volinfo);
 
-        if (ret)
-                goto out;
-
-        ret = dict_get_uint64 (rsp_dict, "files", &files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get file count");
-
-        ret = dict_get_uint64 (rsp_dict, "size", &size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get size of xfer");
-
-        ret = dict_get_uint64 (rsp_dict, "lookups", &lookup);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get lookedup file count");
-        ret = dict_get_int32 (rsp_dict, "status", (int32_t *)&status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "failed to get status");
-
-        volinfo->rebalance_files += files;
-        volinfo->rebalance_data += size;
-        volinfo->lookedup_files += lookup;
+                case GD_OP_SYS_EXEC:
+                        ret = glusterd_op_sys_exec (dict, op_errstr, rsp_dict);
+                        break;
 
-        if (!op_ctx) {
-                dict_copy (rsp_dict, op_ctx);
-                goto out;
+                default:
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown op %s",
+                                gd_op_list[op]);
+                        break;
         }
 
-        ret = dict_set_uint64 (op_ctx, "files", volinfo->rebalance_files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set file count");
-
-        ret = dict_set_uint64 (op_ctx, "size", volinfo->rebalance_data);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set size of xfer");
-
-        ret = dict_set_uint64 (op_ctx, "lookups", volinfo->lookedup_files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set lookedup file count");
-        ret = dict_set_int32 (op_ctx, "status", status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set status");
+        if (ret == 0)
+            glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_POST);
 
-out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-int32_t
-glusterd_handle_node_rsp (glusterd_req_ctx_t *req_ctx, void *pending_entry,
-                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
-                          char **op_errstr)
-{
-        int                     ret = 0;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-
-        GF_ASSERT (op_errstr);
-
-        switch (op) {
-        case GD_OP_PROFILE_VOLUME:
-                brickinfo = pending_entry;
-                ret = glusterd_profile_volume_brick_rsp (brickinfo, rsp_dict,
-                                                         op_ctx, op_errstr);
-                break;
-        case GD_OP_STATUS_VOLUME:
-                brickinfo = pending_entry;
-                ret = glusterd_status_volume_brick_rsp (brickinfo, rsp_dict,
-                                                        op_ctx, op_errstr);
-                break;
-
-        case GD_OP_DEFRAG_BRICK_VOLUME:
-                glusterd_defrag_volume_node_rsp (req_ctx->dict,
-                                                 rsp_dict, op_ctx);
-                break;
-
-        case GD_OP_HEAL_VOLUME:
-                ret = glusterd_heal_volume_brick_rsp (req_ctx->dict, rsp_dict,
-                                                      op_ctx, op_errstr);
-                break;
-        default:
-                break;
-        }
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
 
 static int
-glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr,
+                                    struct list_head *selected)
 {
         int                                     ret = 0;
         int                                     flags = 0;
@@ -2835,15 +4773,17 @@ glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr)
         glusterd_brickinfo_t                    *brickinfo = NULL;
         glusterd_pending_node_t                 *pending_node = NULL;
 
-
         ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
         if (ret)
                 goto out;
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
                 goto out;
+        }
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 if (glusterd_is_brick_started (brickinfo)) {
@@ -2855,7 +4795,7 @@ glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr)
                         } else {
                                 pending_node->node = brickinfo;
                                 pending_node->type = GD_NODE_BRICK;
-                                list_add_tail (&pending_node->list, &opinfo.pending_bricks);
+                                list_add_tail (&pending_node->list, selected);
                                 pending_node = NULL;
                         }
                 }
@@ -2866,7 +4806,8 @@ out:
 }
 
 static int
-glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
+                                     struct list_head *selected)
 {
         int                                     ret = -1;
         char                                    *volname = NULL;
@@ -2879,6 +4820,8 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr)
         glusterd_pending_node_t                 *pending_node = NULL;
         int32_t                                 force = 0;
 
+
+
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
@@ -2915,8 +4858,7 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr)
                 }
 
                 ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_COMPLETE);
+                                                              &brickinfo);
                 if (ret)
                         goto out;
                 if (glusterd_is_brick_started (brickinfo)) {
@@ -2928,7 +4870,7 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr)
                         } else {
                                 pending_node->node = brickinfo;
                                 pending_node->type = GD_NODE_BRICK;
-                                list_add_tail (&pending_node->list, &opinfo.pending_bricks);
+                                list_add_tail (&pending_node->list, selected);
                                 pending_node = NULL;
                         }
                 }
@@ -2940,7 +4882,8 @@ out:
 }
 
 static int
-glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr,
+                                       struct list_head *selected)
 {
         int                                     ret = -1;
         char                                    *volname = NULL;
@@ -2953,6 +4896,8 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
         glusterd_pending_node_t                 *pending_node = NULL;
         char                                    *brick = NULL;
 
+
+
         this = THIS;
         GF_ASSERT (this);
         priv = this->private;
@@ -2987,6 +4932,29 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                 goto out;
                 break;
         case GF_CLI_STATS_INFO:
+                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+                if (ret) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "NFS server"
+                                        " is not running");
+                                goto out;
+                        }
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        }
+                        pending_node->node = priv->nfs;
+                        pending_node->type = GD_NODE_NFS;
+                        list_add_tail (&pending_node->list, selected);
+                        pending_node = NULL;
+
+                        ret = 0;
+                        goto out;
+
+                }
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         if (glusterd_is_brick_started (brickinfo)) {
                                 pending_node = GF_CALLOC (1, sizeof (*pending_node),
@@ -2998,7 +4966,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                                         pending_node->node = brickinfo;
                                         pending_node->type = GD_NODE_BRICK;
                                         list_add_tail (&pending_node->list,
-                                                       &opinfo.pending_bricks);
+                                                       selected);
                                         pending_node = NULL;
                                 }
                         }
@@ -3006,11 +4974,33 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                 break;
 
         case GF_CLI_STATS_TOP:
+                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+                if (ret) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "NFS server"
+                                        " is not running");
+                                goto out;
+                        }
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        }
+                        pending_node->node = priv->nfs;
+                        pending_node->type = GD_NODE_NFS;
+                        list_add_tail (&pending_node->list, selected);
+                        pending_node = NULL;
+
+                        ret = 0;
+                        goto out;
+
+                }
                 ret = dict_get_str (dict, "brick", &brick);
                 if (!ret) {
                         ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                                      &brickinfo,
-                                                                      GF_PATH_COMPLETE);
+                                                                      &brickinfo);
                         if (ret)
                                 goto out;
 
@@ -3026,7 +5016,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                                 pending_node->node = brickinfo;
                                 pending_node->type = GD_NODE_BRICK;
                                 list_add_tail (&pending_node->list,
-                                               &opinfo.pending_bricks);
+                                               selected);
                                 pending_node = NULL;
                                 goto out;
                         }
@@ -3043,7 +5033,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
                                         pending_node->node = brickinfo;
                                         pending_node->type = GD_NODE_BRICK;
                                         list_add_tail (&pending_node->list,
-                                                       &opinfo.pending_bricks);
+                                                       selected);
                                         pending_node = NULL;
                                 }
                         }
@@ -3088,24 +5078,95 @@ out:
 }
 
 int
+get_replica_index_for_per_replica_cmd (glusterd_volinfo_t *volinfo,
+                                       dict_t *dict) {
+        int                     ret = 0;
+        char                    *hostname = NULL;
+        char                    *path = NULL;
+        int                     index = 0;
+        glusterd_brickinfo_t   *brickinfo = NULL;
+        int                     cmd_replica_index = -1;
+        int                     replica_count = -1;
+
+
+        if (!dict)  {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "per-replica-cmd-hostname", &hostname);
+        if (ret)
+                goto out;
+        ret = dict_get_str (dict, "per-replica-cmd-path", &path);
+        if (ret)
+                goto out;
+
+        replica_count = volinfo->replica_count;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+                if (!strcmp (brickinfo->path,  path) &&
+                    !strcmp (brickinfo->hostname, hostname)) {
+                        cmd_replica_index = index/(replica_count);
+                        goto out;
+                }
+                index++;
+        }
+
+
+out:
+        if (ret)
+                cmd_replica_index = -1;
+
+        return cmd_replica_index;
+}
+
+int
 _select_rxlators_with_local_bricks (xlator_t *this, glusterd_volinfo_t *volinfo,
-                                    dict_t *dict)
+                                    dict_t *dict, cli_cmd_type type)
 {
         glusterd_brickinfo_t    *brickinfo = NULL;
         glusterd_conf_t         *priv   = NULL;
-        int                     index = 1;
+        int                     index = 0;
         int                     rxlator_count = 0;
         int                     replica_count = 0;
         gf_boolean_t            add     = _gf_false;
+        int                     ret = 0;
+        int                     cmd_replica_index = -1;
 
         priv = this->private;
         replica_count = volinfo->replica_count;
+
+        if (type == PER_REPLICA) {
+
+                cmd_replica_index = get_replica_index_for_per_replica_cmd
+                                    (volinfo, dict);
+                if (cmd_replica_index == -1) {
+                        ret = -1;
+                        goto err;
+                }
+        }
+
+        index = 1;
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 if (uuid_is_null (brickinfo->uuid))
                         (void)glusterd_resolve_brick (brickinfo);
 
-                if (!uuid_compare (priv->uuid, brickinfo->uuid))
-                        add = _gf_true;
+                switch (type) {
+                        case ALL_REPLICA:
+                                if (!uuid_compare (MY_UUID, brickinfo->uuid))
+                                        add = _gf_true;
+                                break;
+                        case PER_REPLICA:
+                                if (!uuid_compare (MY_UUID, brickinfo->uuid) &&
+                                 ((index-1)/replica_count == cmd_replica_index))
+
+                                                add = _gf_true;
+                                break;
+                }
+
                 if (index % replica_count == 0) {
                         if (add) {
                                 _add_rxlator_to_dict (dict, volinfo->volname,
@@ -3118,6 +5179,10 @@ _select_rxlators_with_local_bricks (xlator_t *this, glusterd_volinfo_t *volinfo,
 
                 index++;
         }
+err:
+        if (ret)
+                rxlator_count = -1;
+
         return rxlator_count;
 }
 
@@ -3144,7 +5209,7 @@ _select_rxlators_for_full_self_heal (xlator_t *this,
                         uuid_copy (candidate, brickinfo->uuid);
 
                 if (index % replica_count == 0) {
-                        if (!uuid_compare (priv->uuid, candidate)) {
+                        if (!uuid_compare (MY_UUID, candidate)) {
                                 _add_rxlator_to_dict (dict, volinfo->volname,
                                                       (index-1)/replica_count,
                                                       rxlator_count);
@@ -3158,8 +5223,80 @@ _select_rxlators_for_full_self_heal (xlator_t *this,
         return rxlator_count;
 }
 
+
 static int
-glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr)
+fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo,
+                                  cli_cmd_type type, dict_t *req_dict)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        char                    msg[1024] = {0,};
+        char                    key[1024]  = {0,};
+        char                    value[1024] = {0,};
+        int                     index = 0;
+        int                     ret = 0;
+        xlator_t               *this = NULL;
+        int                     cmd_replica_index = -1;
+
+        this = THIS;
+        snprintf (msg, sizeof (msg), "self-heal-daemon is not running on");
+
+        if (type == PER_REPLICA) {
+                cmd_replica_index = get_replica_index_for_per_replica_cmd
+                                    (volinfo, req_dict);
+                if (cmd_replica_index == -1) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Could not find the "
+                                "replica index for per replica type command");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+
+                if (uuid_compare (MY_UUID, brickinfo->uuid)) {
+                        index++;
+                        continue;
+                }
+
+                if (type == PER_REPLICA) {
+                      if (cmd_replica_index != (index/volinfo->replica_count)) {
+                              index++;
+                              continue;
+                        }
+
+                }
+                snprintf (key, sizeof (key), "%d-status",index);
+                snprintf (value, sizeof (value), "%s %s",msg,
+                          uuid_utoa(MY_UUID));
+                ret = dict_set_dynstr (dict, key, gf_strdup(value));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                "set the dictionary for shd status msg");
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "%d-shd-status",index);
+                ret = dict_set_str (dict, key, "off");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                " set dictionary for shd status msg");
+                        goto out;
+                }
+
+                index++;
+        }
+
+out:
+        return ret;
+
+}
+
+
+static int
+glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr,
+                                    struct list_head *selected,
+                                    dict_t *rsp_dict)
 {
         int                                     ret = -1;
         char                                    *volname = NULL;
@@ -3199,19 +5336,79 @@ glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr)
         }
 
         switch (heal_op) {
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        if (!rsp_dict) {
+                                gf_log (this->name, GF_LOG_ERROR, "Received "
+                                        "empty ctx.");
+                                goto out;
+                        }
+
+                        ret = fill_shd_status_for_local_bricks (rsp_dict,
+                                                                volinfo,
+                                                                ALL_REPLICA,
+                                                                dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "fill the shd status for the local "
+                                        "bricks");
+                        goto out;
+
+                }
+                break;
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        if (!rsp_dict) {
+                                gf_log (this->name, GF_LOG_ERROR, "Received "
+                                        "empty ctx.");
+                                goto out;
+                        }
+                        ret = fill_shd_status_for_local_bricks (rsp_dict,
+                                                                volinfo,
+                                                                PER_REPLICA,
+                                                                dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "fill the shd status for the local"
+                                        " bricks.");
+                        goto out;
+
+                }
+                break;
+                default:
+                        break;
+        }
+
+
+        switch (heal_op) {
         case GF_AFR_OP_HEAL_FULL:
                 rxlator_count = _select_rxlators_for_full_self_heal (this,
                                                                      volinfo,
                                                                      dict);
                 break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                rxlator_count = _select_rxlators_with_local_bricks (this,
+                                                                   volinfo,
+                                                                   dict,
+                                                                   PER_REPLICA);
+                break;
         default:
                 rxlator_count = _select_rxlators_with_local_bricks (this,
                                                                     volinfo,
-                                                                    dict);
+                                                                    dict,
+                                                                   ALL_REPLICA);
                 break;
         }
         if (!rxlator_count)
                 goto out;
+        if (rxlator_count == -1){
+                gf_log (this->name, GF_LOG_ERROR, "Could not determine the"
+                        "translator count");
+                ret = -1;
+                goto out;
+        }
+
         ret = dict_set_int32 (dict, "count", rxlator_count);
         if (ret)
                 goto out;
@@ -3224,8 +5421,7 @@ glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr)
         } else {
                 pending_node->node = priv->shd;
                 pending_node->type = GD_NODE_SHD;
-                list_add_tail (&pending_node->list,
-                               &opinfo.pending_bricks);
+                list_add_tail (&pending_node->list, selected);
                 pending_node = NULL;
         }
 
@@ -3235,9 +5431,9 @@ out:
 
 }
 
-
 static int
-glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
+                                         struct list_head *selected)
 {
         int                                     ret = -1;
         char                                    *volname = NULL;
@@ -3249,7 +5445,6 @@ glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr)
         this = THIS;
         GF_ASSERT (this);
 
-
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
                 gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
@@ -3282,11 +5477,9 @@ out:
         return ret;
 }
 
-
-
-
 static int
-glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr,
+                                      struct list_head *selected)
 {
         int                     ret = -1;
         int                     cmd = 0;
@@ -3308,7 +5501,7 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
 
         ret = dict_get_int32 (dict, "cmd", &cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get status type");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get status type");
                 goto out;
         }
 
@@ -3321,13 +5514,16 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
         case GF_CLI_STATUS_INODE:
         case GF_CLI_STATUS_FD:
         case GF_CLI_STATUS_CALLPOOL:
+        case GF_CLI_STATUS_NFS:
+        case GF_CLI_STATUS_SHD:
+        case GF_CLI_STATUS_QUOTAD:
                 break;
         default:
                 goto out;
         }
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get volname");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volname");
                 goto out;
         }
         ret = glusterd_volinfo_find (volname, &volinfo);
@@ -3338,18 +5534,17 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
         if ( (cmd & GF_CLI_STATUS_BRICK) != 0) {
                 ret = dict_get_str (dict, "brick", &brickname);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Unable to get brick");
                         goto out;
                 }
                 ret = glusterd_volume_brickinfo_get_by_brick (brickname,
                                                               volinfo,
-                                                              &brickinfo,
-                                                              GF_PATH_COMPLETE);
+                                                              &brickinfo);
                 if (ret)
                         goto out;
 
-                if (uuid_compare (brickinfo->uuid, priv->uuid)||
+                if (uuid_compare (brickinfo->uuid, MY_UUID)||
                     !glusterd_is_brick_started (brickinfo))
                         goto out;
 
@@ -3362,13 +5557,70 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
                 pending_node->node = brickinfo;
                 pending_node->type = GD_NODE_BRICK;
                 pending_node->index = 0;
-                list_add_tail (&pending_node->list, &opinfo.pending_bricks);
+                list_add_tail (&pending_node->list, selected);
+
+                ret = 0;
+        } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                if (!glusterd_is_nodesvc_online ("nfs")) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "NFS server is not running");
+                        goto out;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = priv->nfs;
+                pending_node->type = GD_NODE_NFS;
+                pending_node->index = 0;
+                list_add_tail (&pending_node->list, selected);
+
+                ret = 0;
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Self-heal daemon is not running");
+                        goto out;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = priv->shd;
+                pending_node->type = GD_NODE_SHD;
+                pending_node->index = 0;
+                list_add_tail (&pending_node->list, selected);
+
+                ret = 0;
+        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+                if (!glusterd_is_nodesvc_online ("quotad")) {
+                        gf_log (this->name, GF_LOG_ERROR, "Quotad is not "
+                                "running");
+                        ret = -1;
+                        goto out;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = priv->quotad;
+                pending_node->type = GD_NODE_QUOTAD;
+                pending_node->index = 0;
+                list_add_tail (&pending_node->list, selected);
 
                 ret = 0;
         } else {
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         brick_index++;
-                        if (uuid_compare (brickinfo->uuid, priv->uuid) ||
+                        if (uuid_compare (brickinfo->uuid, MY_UUID) ||
                             !glusterd_is_brick_started (brickinfo)) {
                                 continue;
                         }
@@ -3383,8 +5635,7 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
                         pending_node->node = brickinfo;
                         pending_node->type = GD_NODE_BRICK;
                         pending_node->index = brick_index;
-                        list_add_tail (&pending_node->list,
-                                       &opinfo.pending_bricks);
+                        list_add_tail (&pending_node->list, selected);
                         pending_node = NULL;
                 }
         }
@@ -3401,6 +5652,7 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
         xlator_t                        *this = NULL;
         glusterd_op_t                   op = GD_OP_NONE;
         glusterd_req_ctx_t              *req_ctx = NULL;
+        char                            *op_errstr = NULL;
 
         this = THIS;
         priv = this->private;
@@ -3412,10 +5664,18 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
                                      gf_gld_mt_op_allack_ctx_t);
                 op = glusterd_op_get_op ();
                 req_ctx->op = op;
-                uuid_copy (req_ctx->uuid, priv->uuid);
-                ret = glusterd_op_build_payload (&req_ctx->dict);
-                if (ret)//TODO:what to do??
+                uuid_copy (req_ctx->uuid, MY_UUID);
+                ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr,
+                                                 NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                                gd_op_list[op]);
+                        if (op_errstr == NULL)
+                                gf_asprintf (&op_errstr,
+                                             OPERRSTR_BUILD_PAYLOAD);
+                        opinfo.op_errstr = op_errstr;
                         goto out;
+                }
         }
 
         proc = &priv->gfs_mgmt->proctable[GLUSTERD_BRICK_OP];
@@ -3427,11 +5687,12 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
 
         if (!opinfo.pending_count && !opinfo.brick_pending_count) {
                 glusterd_clear_pending_nodes (&opinfo.pending_bricks);
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, req_ctx);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                                   &event->txn_id, req_ctx);
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
@@ -3444,10 +5705,14 @@ glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
         char                        *op_errstr = NULL;
         glusterd_op_t               op = GD_OP_NONE;
+        gd_node_type                type = GD_NODE_NONE;
         dict_t                      *op_ctx = NULL;
         glusterd_req_ctx_t          *req_ctx = NULL;
         void                        *pending_entry = NULL;
+        xlator_t                    *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (event);
         GF_ASSERT (ctx);
         ev_ctx = ctx;
@@ -3458,11 +5723,12 @@ glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         op = req_ctx->op;
         op_ctx = glusterd_op_get_ctx ();
         pending_entry = ev_ctx->pending_node->node;
+        type = ev_ctx->pending_node->type;
 
         ret = glusterd_remove_pending_entry (&opinfo.pending_bricks,
                                              pending_entry);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "unknown response received ");
+                gf_log (this->name, GF_LOG_ERROR, "unknown response received ");
                 ret = -1;
                 goto out;
         }
@@ -3470,25 +5736,27 @@ glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.brick_pending_count > 0)
                 opinfo.brick_pending_count--;
 
-        glusterd_handle_node_rsp (req_ctx, pending_entry, op, ev_ctx->rsp_dict,
-                                  op_ctx, &op_errstr);
+        glusterd_handle_node_rsp (req_ctx->dict, pending_entry, op, ev_ctx->rsp_dict,
+                                  op_ctx, &op_errstr, type);
 
         if (opinfo.brick_pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, ev_ctx->commit_ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                           ev_ctx->commit_ctx);
 
 out:
         if (ev_ctx->rsp_dict)
                 dict_unref (ev_ctx->rsp_dict);
         GF_FREE (ev_ctx);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
 
 int32_t
-glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr)
+glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           struct list_head *selected, dict_t *rsp_dict)
 {
         int     ret = 0;
 
@@ -3499,33 +5767,40 @@ glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr)
 
         switch (op) {
         case GD_OP_STOP_VOLUME:
-                ret = glusterd_bricks_select_stop_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_stop_volume (dict, op_errstr,
+                                                          selected);
                 break;
 
         case GD_OP_REMOVE_BRICK:
-                ret = glusterd_bricks_select_remove_brick (dict, op_errstr);
+                ret = glusterd_bricks_select_remove_brick (dict, op_errstr,
+                                                           selected);
                 break;
 
         case GD_OP_PROFILE_VOLUME:
-                ret = glusterd_bricks_select_profile_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_profile_volume (dict, op_errstr,
+                                                             selected);
                 break;
 
         case GD_OP_HEAL_VOLUME:
-                ret = glusterd_bricks_select_heal_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_heal_volume (dict, op_errstr,
+                                                          selected, rsp_dict);
                 break;
 
         case GD_OP_STATUS_VOLUME:
-                ret = glusterd_bricks_select_status_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_status_volume (dict, op_errstr,
+                                                            selected);
                 break;
 
         case GD_OP_DEFRAG_BRICK_VOLUME:
-                ret = glusterd_bricks_select_rebalance_volume (dict, op_errstr);
+                ret = glusterd_bricks_select_rebalance_volume (dict, op_errstr,
+                                                               selected);
                 break;
+
         default:
                 break;
          }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -3842,7 +6117,7 @@ glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
 
 int
 glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             void *ctx)
+                             uuid_t *txn_id, void *ctx)
 {
         int32_t                 ret = -1;
         glusterd_op_sm_event_t  *event = NULL;
@@ -3857,7 +6132,10 @@ glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
 
         event->ctx = ctx;
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Enqueue event: '%s'",
+        if (txn_id)
+                uuid_copy (event->txn_id, *txn_id);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Enqueue event: '%s'",
                 glusterd_op_sm_event_name_get (event->event));
         list_add_tail (&event->list, &gd_op_sm_queue);
 
@@ -3916,9 +6194,14 @@ glusterd_op_sm ()
         glusterd_op_sm_ac_fn            handler = NULL;
         glusterd_op_sm_t                *state = NULL;
         glusterd_op_sm_event_type_t     event_type = GD_OP_EVENT_NONE;
+        xlator_t                        *this = NULL;
+        glusterd_op_info_t              txn_op_info;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         if ((lock_err = pthread_mutex_trylock (&gd_op_sm_lock))) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "lock failed due to %s",
+                gf_log (this->name, GF_LOG_ERROR, "lock failed due to %s",
                         strerror (lock_err));
                 goto lock_failed;
         }
@@ -3929,9 +6212,24 @@ glusterd_op_sm ()
 
                         list_del_init (&event->list);
                         event_type = event->event;
-                        gf_log ("", GF_LOG_DEBUG, "Dequeued event of type: '%s'",
+                        gf_log (this->name, GF_LOG_DEBUG, "Dequeued event of "
+                                "type: '%s'",
                                 glusterd_op_sm_event_name_get(event_type));
 
+                        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s",
+                                uuid_utoa (event->txn_id));
+
+                        ret = glusterd_get_txn_opinfo (&event->txn_id,
+                                                       &txn_op_info);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to get transaction's opinfo");
+                                glusterd_destroy_op_event_ctx (event);
+                                GF_FREE (event);
+                                continue;
+                        } else
+                                opinfo = txn_op_info;
+
                         state = glusterd_op_state_table[opinfo.state.state];
 
                         GF_ASSERT (state);
@@ -3942,7 +6240,7 @@ glusterd_op_sm ()
                         ret = handler (event, event->ctx);
 
                         if (ret) {
-                                gf_log ("glusterd", GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "handler returned: %d", ret);
                                 glusterd_destroy_op_event_ctx (event);
                                 GF_FREE (event);
@@ -3953,7 +6251,7 @@ glusterd_op_sm ()
                                                                 event_type);
 
                         if (ret) {
-                                gf_log ("glusterd", GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "Unable to transition"
                                         "state from '%s' to '%s'",
                          glusterd_op_sm_state_name_get(opinfo.state.state),
@@ -3962,8 +6260,27 @@ glusterd_op_sm ()
                                 return ret;
                         }
 
+                        if ((state[event_type].next_state ==
+                             GD_OP_STATE_DEFAULT) &&
+                           (event_type == GD_OP_EVENT_UNLOCK)) {
+                                /* Clearing the transaction opinfo */
+                                ret = glusterd_clear_txn_opinfo(&event->txn_id);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to clear "
+                                                "transaction's opinfo");
+                        } else {
+                                ret = glusterd_set_txn_opinfo (&event->txn_id,
+                                                               &opinfo);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to set "
+                                                "transaction's opinfo");
+                        }
+
                         glusterd_destroy_op_event_ctx (event);
                         GF_FREE (event);
+
                 }
         }
 
@@ -4017,48 +6334,6 @@ glusterd_op_clear_op (glusterd_op_t op)
 }
 
 int32_t
-glusterd_op_init_ctx (glusterd_op_t op)
-{
-        int     ret = 0;
-        dict_t *dict = NULL;
-
-        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
-
-        if (_gf_false == glusterd_need_brick_op (op)) {
-                gf_log ("", GF_LOG_DEBUG, "Received op: %d, returning", op);
-                goto out;
-        }
-        dict = dict_new ();
-        if (dict == NULL) {
-                ret = -1;
-                goto out;
-        }
-        ret = glusterd_op_set_ctx (dict);
-        if (ret)
-                goto out;
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-
-
-int32_t
-glusterd_op_fini_ctx ()
-{
-        dict_t *dict = NULL;
-
-        dict = glusterd_op_get_ctx ();
-        if (dict)
-                dict_unref (dict);
-
-        glusterd_op_reset_ctx ();
-        return 0;
-}
-
-
-
-int32_t
 glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
 {
 
@@ -4112,4 +6387,3 @@ glusterd_op_sm_init ()
         pthread_mutex_init (&gd_op_sm_lock, NULL);
         return 0;
 }
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index b4df82017..4a73b08f4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_OP_SM_H_
 #define _GLUSTERD_OP_SM_H_
 
@@ -25,8 +15,8 @@
 #include "config.h"
 #endif
 
-#ifndef GSYNC_CONF
-#define GSYNC_CONF GEOREP"/gsyncd.conf"
+#ifndef GSYNC_CONF_TEMPLATE
+#define GSYNC_CONF_TEMPLATE GEOREP"/gsyncd_template.conf"
 #endif
 
 #include <pthread.h>
@@ -42,6 +32,8 @@
 #include "protocol-common.h"
 
 #define GD_VOLUME_NAME_MAX 256
+#define GD_OP_PROTECTED    (0x02)
+#define GD_OP_UNPROTECTED  (0x04)
 
 typedef enum glusterd_op_sm_state_ {
         GD_OP_STATE_DEFAULT = 0,
@@ -85,6 +77,7 @@ struct glusterd_op_sm_event_ {
         struct list_head                list;
         void                            *ctx;
         glusterd_op_sm_event_type_t     event;
+        uuid_t                          txn_id;
 };
 
 typedef struct glusterd_op_sm_event_ glusterd_op_sm_event_t;
@@ -127,6 +120,7 @@ typedef struct glusterd_op_log_filename_ctx_ glusterd_op_log_filename_ctx_t;
 
 struct glusterd_op_lock_ctx_ {
         uuid_t                  uuid;
+        dict_t                 *dict;
         rpcsvc_request_t        *req;
 };
 
@@ -157,23 +151,42 @@ typedef struct glusterd_pr_brick_rsp_conv_t {
 typedef struct glusterd_heal_rsp_conv_ {
         dict_t *dict;
         glusterd_volinfo_t *volinfo;
+        xlator_t *this;
 } glusterd_heal_rsp_conv_t;
 
 typedef struct glusterd_status_rsp_conv_ {
         int count;
+        int brick_index_max;
+        int other_count;
         dict_t *dict;
 } glusterd_status_rsp_conv_t;
 
 typedef struct glusterd_gsync_status_temp {
         dict_t *rsp_dict;
         glusterd_volinfo_t *volinfo;
+        char *node;
 }glusterd_gsync_status_temp_t;
+
+typedef struct gsync_status_param {
+        int is_active;
+        glusterd_volinfo_t *volinfo;
+}gsync_status_param_t;
+
+typedef struct glusterd_txn_opinfo_object_ {
+        glusterd_op_info_t    opinfo;
+} glusterd_txn_opinfo_obj;
+
+typedef enum cli_cmd_type_ {
+        PER_REPLICA,
+        ALL_REPLICA,
+ } cli_cmd_type;
+
 int
 glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
                           glusterd_op_sm_event_t **new_event);
 int
 glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             void *ctx);
+                             uuid_t *txn_id, void *ctx);
 
 int
 glusterd_op_sm_init ();
@@ -188,7 +201,7 @@ int32_t
 glusterd_op_set_op (glusterd_op_t op);
 
 int
-glusterd_op_build_payload (dict_t **req);
+glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx);
 
 int32_t
 glusterd_op_stage_validate (glusterd_op_t op, dict_t *req, char **op_errstr,
@@ -198,6 +211,13 @@ int32_t
 glusterd_op_commit_perform (glusterd_op_t op, dict_t *req, char **op_errstr,
                             dict_t* dict);
 
+int32_t
+glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                       char *err_str, size_t err_len);
+
+int32_t
+glusterd_op_txn_complete ();
+
 void *
 glusterd_op_get_ctx ();
 
@@ -225,8 +245,6 @@ set_xlator_option (dict_t *dict, char *key, char *value);
 
 void
 glusterd_do_replace_brick (void *data);
-int
-glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key, int32_t is_force);
 
 char*
 glusterd_op_sm_state_name_get (int state);
@@ -234,18 +252,25 @@ glusterd_op_sm_state_name_get (int state);
 char*
 glusterd_op_sm_event_name_get (int event);
 int32_t
-glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr);
+glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           struct list_head *selected, dict_t *rsp_dict);
 int
 glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
                                  gd1_mgmt_brick_op_req **req, dict_t *dict);
+int
+glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+                               dict_t *dict);
 int32_t
-glusterd_handle_brick_rsp (glusterd_brickinfo_t *brickinfo,
-                           glusterd_op_t op, dict_t *rsp_dict, dict_t *ctx_dict,
-                           char **op_errstr);
-int32_t
-glusterd_op_init_ctx (glusterd_op_t op);
-int32_t
-glusterd_op_fini_ctx ();
+glusterd_handle_brick_rsp (void *pending_entry, glusterd_op_t op,
+                           dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
+                           gd_node_type type);
+
+dict_t*
+glusterd_op_init_commit_rsp_dict (glusterd_op_t op);
+
+void
+glusterd_op_modify_op_ctx (glusterd_op_t op, void *op_ctx);
+
 int32_t
 glusterd_volume_stats_read_perf (char *brick_path, int32_t blk_size,
                 int32_t blk_count, double *throughput, double *time);
@@ -261,14 +286,29 @@ glusterd_are_all_volumes_stopped ();
 int
 glusterd_stop_bricks (glusterd_volinfo_t *volinfo);
 int
-glusterd_get_gsync_status_mst_slv( glusterd_volinfo_t *volinfo,
-                                   char *slave, dict_t *rsp_dict);
+gsync_status (char *master, char *slave, char *conf_path, int *status);
+
 int
-gsync_status (char *master, char *slave, int *status);
+glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag);
 
 int
-glusterd_gsync_get_param_file (char *prmfile, const char *ext, char *master,
-                                char *slave, char *gl_workdir);
+glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                 dict_t *op_ctx);
+#ifdef HAVE_BD_XLATOR
 int
-glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag);
+glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg);
+#endif
+
+int32_t
+glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+
+int32_t
+glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+
+int32_t
+glusterd_clear_txn_opinfo (uuid_t *txn_id);
+
+int32_t
+glusterd_generate_txn_id (dict_t *dict, uuid_t **txn_id);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
index 08e7aedca..a153ca1a9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.c
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -62,8 +52,8 @@ pmap_port_isfree (int port)
 }
 
 
-struct pmap_registry *
-pmap_registry_new (void)
+static struct pmap_registry *
+pmap_registry_new (xlator_t *this)
 {
         struct pmap_registry *pmap = NULL;
         int                   i = 0;
@@ -79,8 +69,8 @@ pmap_registry_new (void)
                         pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
         }
 
-        pmap->base_port = GF_DEFAULT_BASE_PORT + 2;
-        pmap->last_alloc = GF_DEFAULT_BASE_PORT + 2;
+        pmap->base_port = pmap->last_alloc =
+                ((glusterd_conf_t *)(this->private))->base_port;
 
         return pmap;
 }
@@ -96,7 +86,7 @@ pmap_registry_get (xlator_t *this)
 
         pmap = priv->pmap;
         if (!pmap) {
-                pmap = pmap_registry_new ();
+                pmap = pmap_registry_new (this);
                 if (!pmap)
                         return NULL;
                 priv->pmap = pmap;
@@ -228,8 +218,7 @@ pmap_registry_bind (xlator_t *this, int port, const char *brickname,
 
         p = port;
         pmap->ports[p].type = type;
-        if (pmap->ports[p].brickname)
-                free (pmap->ports[p].brickname);
+        free (pmap->ports[p].brickname);
         pmap->ports[p].brickname = strdup (brickname);
         pmap->ports[p].type = type;
         pmap->ports[p].xprt = xprt;
@@ -281,8 +270,7 @@ remove:
         gf_log ("pmap", GF_LOG_INFO, "removing brick %s on port %d",
                 pmap->ports[p].brickname, p);
 
-        if (pmap->ports[p].brickname)
-                free (pmap->ports[p].brickname);
+        free (pmap->ports[p].brickname);
 
         pmap->ports[p].brickname = NULL;
         pmap->ports[p].xprt = NULL;
@@ -292,15 +280,17 @@ out:
 }
 
 int
-gluster_pmap_portbybrick (rpcsvc_request_t *req)
+__gluster_pmap_portbybrick (rpcsvc_request_t *req)
 {
         pmap_port_by_brick_req    args = {0,};
         pmap_port_by_brick_rsp    rsp  = {0,};
         char                     *brick = NULL;
         int                       port = 0;
+        int                       ret = -1;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_port_by_brick_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_port_by_brick_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
@@ -317,21 +307,29 @@ gluster_pmap_portbybrick (rpcsvc_request_t *req)
 fail:
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        if (args.brick)
-                free (args.brick);//malloced by xdr
+        free (args.brick);//malloced by xdr
 
         return 0;
 }
 
 
 int
-gluster_pmap_brickbyport (rpcsvc_request_t *req)
+gluster_pmap_portbybrick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_portbybrick);
+}
+
+
+int
+__gluster_pmap_brickbyport (rpcsvc_request_t *req)
 {
         pmap_brick_by_port_req    args = {0,};
         pmap_brick_by_port_rsp    rsp  = {0,};
+        int                       ret = -1;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_brick_by_port_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_brick_by_port_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
@@ -349,6 +347,14 @@ fail:
         return 0;
 }
 
+
+int
+gluster_pmap_brickbyport (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_brickbyport);
+}
+
+
 static int
 glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
                               gf_boolean_t value)
@@ -359,14 +365,16 @@ glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
 }
 
 int
-gluster_pmap_signup (rpcsvc_request_t *req)
+__gluster_pmap_signup (rpcsvc_request_t *req)
 {
         pmap_signup_req    args = {0,};
         pmap_signup_rsp    rsp  = {0,};
+        int                ret = -1;
 
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_signup_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signup_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
@@ -377,22 +385,28 @@ gluster_pmap_signup (rpcsvc_request_t *req)
 fail:
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_pmap_signup_rsp);
-        if (args.brick)
-                free (args.brick);//malloced by xdr
+        free (args.brick);//malloced by xdr
 
         return 0;
 }
 
 int
-gluster_pmap_signin (rpcsvc_request_t *req)
+gluster_pmap_signup (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signup);
+}
+
+int
+__gluster_pmap_signin (rpcsvc_request_t *req)
 {
         pmap_signin_req    args = {0,};
         pmap_signin_rsp    rsp  = {0,};
         glusterd_brickinfo_t *brickinfo = NULL;
         int                ret = -1;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_signin_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signin_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
@@ -405,8 +419,7 @@ gluster_pmap_signin (rpcsvc_request_t *req)
 fail:
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_pmap_signin_rsp);
-        if (args.brick)
-                free (args.brick);//malloced by xdr
+        free (args.brick);//malloced by xdr
 
         if (!ret)
                 glusterd_brick_update_signin (brickinfo, _gf_true);
@@ -415,17 +428,24 @@ fail:
 }
 
 
+int
+gluster_pmap_signin (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signin);
+}
+
 
 int
-gluster_pmap_signout (rpcsvc_request_t *req)
+__gluster_pmap_signout (rpcsvc_request_t *req)
 {
         pmap_signout_req    args = {0,};
         pmap_signout_rsp    rsp  = {0,};
         int                 ret = -1;
         glusterd_brickinfo_t *brickinfo = NULL;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_pmap_signout_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signout_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
@@ -439,8 +459,7 @@ gluster_pmap_signout (rpcsvc_request_t *req)
 fail:
         glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                (xdrproc_t)xdr_pmap_signout_rsp);
-        if (args.brick)
-                free (args.brick);//malloced by xdr
+        free (args.brick);//malloced by xdr
 
         if (!ret)
                 glusterd_brick_update_signin (brickinfo, _gf_false);
@@ -448,18 +467,19 @@ fail:
         return 0;
 }
 
+int
+gluster_pmap_signout (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signout);
+}
+
 rpcsvc_actor_t gluster_pmap_actors[] = {
-        [GF_PMAP_NULL] = {"NULL", GF_PMAP_NULL, NULL, NULL, NULL, 0},
-        [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK,
-                                 gluster_pmap_portbybrick, NULL, NULL, 0},
-        [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", GF_PMAP_BRICKBYPORT,
-                                 gluster_pmap_brickbyport, NULL, NULL, 0},
-        [GF_PMAP_SIGNIN] = {"SIGNIN", GF_PMAP_SIGNIN,
-                              gluster_pmap_signin, NULL, NULL, 0},
-        [GF_PMAP_SIGNOUT] = {"SIGNOUT", GF_PMAP_SIGNOUT,
-                              gluster_pmap_signout, NULL, NULL, 0},
-        [GF_PMAP_SIGNUP] = {"SIGNUP", GF_PMAP_SIGNUP,
-                              gluster_pmap_signup, NULL, NULL, 0},
+        [GF_PMAP_NULL]        = {"NULL",        GF_PMAP_NULL,        NULL,                     NULL, 0, DRC_NA},
+        [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK, gluster_pmap_portbybrick, NULL, 0, DRC_NA},
+        [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", GF_PMAP_BRICKBYPORT, gluster_pmap_brickbyport, NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNIN]      = {"SIGNIN",      GF_PMAP_SIGNIN,      gluster_pmap_signin,      NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNOUT]     = {"SIGNOUT",     GF_PMAP_SIGNOUT,     gluster_pmap_signout,     NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNUP]      = {"SIGNUP",      GF_PMAP_SIGNUP,      gluster_pmap_signup,      NULL, 0, DRC_NA},
 };
 
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h
index 8f1bb4500..6336ee998 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.h
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_PMAP_H_
 #define _GLUSTERD_PMAP_H_
 
@@ -38,6 +28,8 @@
 #include "rpcsvc.h"
 
 
+#define GF_IANA_PRIV_PORTS_START 49152 /* RFC 6335 */
+
 struct pmap_port_status {
         gf_pmap_port_type_t type;
         char  *brickname;
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index b063421ec..cf23b6404 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -31,24 +21,54 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "run.h"
+#include "syscall.h"
+#include "byte-order.h"
+#include "compat-errno.h"
 
 #include <sys/wait.h>
+#include <dlfcn.h>
+
+/* Any negative pid to make it special client */
+#define QUOTA_CRAWL_PID "-100"
+
+const char *gd_quota_op_list[GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT+1] = {
+        [GF_QUOTA_OPTION_TYPE_NONE]               = "none",
+        [GF_QUOTA_OPTION_TYPE_ENABLE]             = "enable",
+        [GF_QUOTA_OPTION_TYPE_DISABLE]            = "disable",
+        [GF_QUOTA_OPTION_TYPE_LIMIT_USAGE]        = "limit-usage",
+        [GF_QUOTA_OPTION_TYPE_REMOVE]             = "remove",
+        [GF_QUOTA_OPTION_TYPE_LIST]               = "list",
+        [GF_QUOTA_OPTION_TYPE_VERSION]            = "version",
+        [GF_QUOTA_OPTION_TYPE_ALERT_TIME]         = "alert-time",
+        [GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT]       = "soft-timeout",
+        [GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT]       = "hard-timeout",
+        [GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT] = "default-soft-limit",
+};
 
 int
-glusterd_handle_quota (rpcsvc_request_t *req)
+glusterd_store_quota_config (glusterd_volinfo_t *volinfo, char *path,
+                             char *gfid_str, int opcode, char **op_errstr);
+int
+__glusterd_handle_quota (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
         dict_t                         *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_QUOTA;
-        char                            operation[256] = {0, };
         char                           *volname = NULL;
         int32_t                         type = 0;
+        char                            msg[2048] = {0,};
+        xlator_t                       *this = NULL;
+        glusterd_conf_t                *conf = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
@@ -62,8 +82,10 @@ glusterd_handle_quota (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                     "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -72,54 +94,49 @@ glusterd_handle_quota (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to get volume name, while"
-                        "handling quota command");
+                snprintf (msg, sizeof (msg), "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name, "
+                        "while handling quota command");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "Unable to get type of cmd. , while"
-                        "handling quota command");
-                goto out;
+                snprintf (msg, sizeof (msg), "Unable to get type of command");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of cmd, "
+                        "while handling quota command");
+               goto out;
         }
 
-        switch (type) {
-        case GF_QUOTA_OPTION_TYPE_ENABLE:
-                strncpy (operation, "enable", sizeof (operation));
-                break;
-
-        case GF_QUOTA_OPTION_TYPE_DISABLE:
-                strncpy (operation, "disable", sizeof (operation));
-                break;
-
-        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                strncpy (operation, "limit-usage", sizeof (operation));
-                break;
+        if ((conf->op_version == GD_OP_VERSION_MIN) &&
+            (type > GF_QUOTA_OPTION_TYPE_VERSION)) {
+                snprintf (msg, sizeof (msg), "Cannot execute command. The "
+                         "cluster is operating at version %d. Quota command %s "
+                         "is unavailable in this version", conf->op_version,
+                         gd_quota_op_list[type]);
+                ret = -1;
+                goto out;
+       }
 
-        case GF_QUOTA_OPTION_TYPE_REMOVE:
-                strncpy (operation, "remove", sizeof (operation));
-                break;
-        }
-        gf_cmd_log ("volume quota", " %s command on %s", operation, volname);
-        ret = glusterd_op_begin (req, GD_OP_QUOTA, dict);
-        gf_cmd_log ("volume quota", " %s command on %s %s", operation,volname,
-                    (ret != 0)? "FAILED" : "SUCCEEDED");
+        ret = glusterd_op_begin_synctask (req, GD_OP_QUOTA, dict);
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, msg);
         }
 
         return ret;
 }
 
+int
+glusterd_handle_quota (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_quota);
+}
+
 int32_t
 glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo)
 {
@@ -134,7 +151,6 @@ glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo)
         }
 
         if (flag == _gf_false) {
-                gf_log ("", GF_LOG_ERROR, "first enable the quota translator");
                 ret = -1;
                 goto out;
         }
@@ -143,108 +159,9 @@ out:
         return ret;
 }
 
-/* At the end of the function, the variable found will be set
- * to true if the path to be removed was present in the limit-list,
- * else will be false.
- */
-int32_t
-_glusterd_quota_remove_limits (char **quota_limits, char *path,
-                               gf_boolean_t *found)
-{
-        int      ret      = 0;
-        int      i        = 0;
-        int      size     = 0;
-        int      len      = 0;
-        int      pathlen  = 0;
-        int      skiplen  = 0;
-        int      flag     = 0;
-        char    *limits   = NULL;
-        char    *qlimits  = NULL;
-
-        if (found != NULL)
-                *found = _gf_false;
-
-        if (*quota_limits == NULL)
-                return -1;
-
-        qlimits = *quota_limits;
-
-        pathlen = strlen (path);
-
-        len = strlen (qlimits);
-
-        limits = GF_CALLOC (len + 1, sizeof (char), gf_gld_mt_char);
-        if (!limits)
-                return -1;
-
-        while (i < len) {
-                if (!memcmp ((void *) &qlimits [i], (void *)path, pathlen))
-                        if (qlimits [i + pathlen] == ':') {
-                                flag = 1;
-                                if (found != NULL)
-                                        *found = _gf_true;
-                        }
-
-                while (qlimits [i + size] != ',' &&
-                       qlimits [i + size] != '\0')
-                        size++;
-
-                if (!flag) {
-                        memcpy ((void *) &limits [i], (void *) &qlimits [i], size + 1);
-                } else {
-                        skiplen = size + 1;
-                        size = len - i - size;
-                        memcpy ((void *) &limits [i], (void *) &qlimits [i + skiplen], size);
-                        break;
-                }
-
-                i += size + 1;
-                size = 0;
-        }
-
-        if (!flag) {
-                ret = 1;
-        } else {
-                len = strlen (limits);
-
-                if (len == 0) {
-                        GF_FREE (qlimits);
-
-                        *quota_limits = NULL;
-
-                        goto out;
-                }
-
-                if (limits[len - 1] == ',') {
-                        limits[len - 1] = '\0';
-                        len --;
-                }
-
-                GF_FREE (qlimits);
-
-                qlimits = GF_CALLOC (len + 1, sizeof (char), gf_gld_mt_char);
-
-                if (!qlimits) {
-                        ret = -1;
-                        goto out;
-                }
-
-                memcpy ((void *) qlimits, (void *) limits, len + 1);
-
-                *quota_limits = qlimits;
-
-                ret = 0;
-        }
-
-out:
-        if (limits)
-                GF_FREE (limits);
-
-        return ret;
-}
-
 int32_t
-glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
+glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
+                                  int type)
 {
         pid_t                      pid;
         int32_t                    ret              = 0;
@@ -263,10 +180,14 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
         runner_add_args (&runner, SBIN_DIR"/glusterfs",
                          "-s", "localhost",
                          "--volfile-id", volname,
+			 "--use-readdirp=no",
+                         "--client-pid", QUOTA_CRAWL_PID,
                          "-l", DEFAULT_LOG_FILE_DIRECTORY"/quota-crawl.log",
                          mountdir, NULL);
 
+        synclock_unlock (&priv->big_lock);
         ret = runner_run_reuse (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret == -1) {
                 runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
                 runner_end (&runner);
@@ -293,7 +214,19 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
                         exit (EXIT_FAILURE);
                 }
                 runinit (&runner);
-                runner_add_args (&runner, "/usr/bin/find", "find", ".", NULL);
+
+                if (type == GF_QUOTA_OPTION_TYPE_ENABLE)
+
+                        runner_add_args (&runner, "/usr/bin/find", "find", ".",
+                                         NULL);
+
+                else if (type == GF_QUOTA_OPTION_TYPE_DISABLE)
+
+                        runner_add_args (&runner, "/usr/bin/find", ".",
+                                         "-exec", "/usr/bin/setfattr", "-n",
+                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "-v",
+                                         "1", "{}", "\\", ";", NULL);
+
                 if (runner_start (&runner) == -1)
                         _exit (EXIT_FAILURE);
 
@@ -313,115 +246,39 @@ out:
         return ret;
 }
 
-char *
-glusterd_quota_get_limit_value (char *quota_limits, char *path)
+int32_t
+glusterd_quota_get_default_soft_limit (glusterd_volinfo_t *volinfo,
+                                       dict_t *rsp_dict)
 {
-        int32_t i, j, k, l, len;
-        int32_t pat_len, diff;
-        char   *ret_str = NULL;
-
-        len = strlen (quota_limits);
-        pat_len = strlen (path);
-        i = 0;
-        j = 0;
-
-        while (i < len) {
-                j = i;
-                k = 0;
-                while (path [k] == quota_limits [j]) {
-                        j++;
-                        k++;
-                }
-
-                l = j;
+        int32_t            ret             = 0;
+        xlator_t          *this            = NULL;
+        glusterd_conf_t   *conf            = NULL;
+        char              *default_limit   = NULL;
+        char              *val             = NULL;
 
-                while (quota_limits [j] != ',' &&
-                       quota_limits [j] != '\0')
-                        j++;
-
-                if (quota_limits [l] == ':' && pat_len == (l - i)) {
-                        diff = j - i;
-                        ret_str = GF_CALLOC (diff + 1, sizeof (char),
-                                             gf_gld_mt_char);
-
-                        strncpy (ret_str, &quota_limits [i], diff);
-
-                        break;
-                }
-                i = ++j; //skip ','
-        }
-
-        return ret_str;
-}
-
-char*
-_glusterd_quota_get_limit_usages (glusterd_volinfo_t *volinfo,
-                                  char *path, char **op_errstr)
-{
-        int32_t  ret          = 0;
-        char    *quota_limits = NULL;
-        char    *ret_str      = NULL;
+        if (rsp_dict == NULL)
+                return -1;
 
-        if (volinfo == NULL)
-                return NULL;
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
 
-        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_LIMIT_USAGE,
-                                    &quota_limits);
-        if (ret)
-                return NULL;
-        if (quota_limits == NULL) {
-                ret_str = NULL;
-                *op_errstr = gf_strdup ("Limit not set on any directory");
-        } else if (path == NULL)
-                ret_str = gf_strdup (quota_limits);
+        ret = glusterd_volinfo_get (volinfo, "features.default-soft-limit",
+                                    &default_limit);
+        if (default_limit)
+                val = gf_strdup (default_limit);
         else
-                ret_str = glusterd_quota_get_limit_value (quota_limits, path);
-
-        return ret_str;
-}
-
-int32_t
-glusterd_quota_get_limit_usages (glusterd_conf_t *priv,
-                                 glusterd_volinfo_t *volinfo,
-                                 char *volname,
-                                 dict_t *dict,
-                                 char **op_errstr)
-{
-        int32_t i               = 0;
-        int32_t ret             = 0;
-        int32_t count           = 0;
-        char    *path           = NULL;
-        dict_t  *ctx            = NULL;
-        char    cmd_str [1024]  = {0, };
-        char   *ret_str         = NULL;
+                val = gf_strdup ("80%");
 
-        ctx = glusterd_op_get_ctx ();
-        if (ctx == NULL)
-                return 0;
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret < 0)
+        ret = dict_set_dynstr (rsp_dict, "default-soft-limit", val);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set default "
+                        "soft-limit into dict");
                 goto out;
-
-        if (count == 0) {
-                ret_str = _glusterd_quota_get_limit_usages (volinfo, NULL,
-                                                            op_errstr);
-        } else {
-                i = 0;
-                while (count--) {
-                        snprintf (cmd_str, 1024, "path%d", i++);
-
-                        ret = dict_get_str (dict, cmd_str, &path);
-                        if (ret < 0)
-                                goto out;
-
-                        ret_str = _glusterd_quota_get_limit_usages (volinfo, path, op_errstr);
-                }
         }
+        ret = 0;
 
-        if (ret_str) {
-                ret = dict_set_dynstr (ctx, "limit_list", ret_str);
-        }
 out:
         return ret;
 }
@@ -432,54 +289,79 @@ glusterd_quota_enable (glusterd_volinfo_t *volinfo, char **op_errstr,
 {
         int32_t         ret     = -1;
         char            *quota_status = NULL;
+        xlator_t        *this         = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
-        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", crawl, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", op_errstr, out);
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, crawl, out);
+        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
 
         if (glusterd_is_volume_started (volinfo) == 0) {
                 *op_errstr = gf_strdup ("Volume is stopped, start volume "
                                         "to enable quota.");
+                ret = -1;
                 goto out;
         }
 
         ret = glusterd_check_if_quota_trans_enabled (volinfo);
         if (ret == 0) {
                 *op_errstr = gf_strdup ("Quota is already enabled");
+                ret = -1;
                 goto out;
         }
 
         quota_status = gf_strdup ("on");
         if (!quota_status) {
-                gf_log ("", GF_LOG_ERROR, "memory allocation failed");
-                *op_errstr = gf_strdup ("Enabling quota has been unsuccessful");
+                gf_log (this->name, GF_LOG_ERROR, "memory allocation failed");
+                ret = -1;
                 goto out;
         }
 
-        ret = dict_set_dynstr (volinfo->dict, VKEY_FEATURES_QUOTA, quota_status);
+        ret = dict_set_dynstr (volinfo->dict, VKEY_FEATURES_QUOTA,
+                               quota_status);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "dict set failed");
-                *op_errstr = gf_strdup ("Enabling quota has been unsuccessful");
+                gf_log (this->name, GF_LOG_ERROR, "dict set failed");
                 goto out;
         }
 
-        *op_errstr = gf_strdup ("Enabling quota has been successful");
-
         *crawl = _gf_true;
 
+        ret = glusterd_store_quota_config (volinfo, NULL, NULL,
+                                           GF_QUOTA_OPTION_TYPE_ENABLE,
+                                           op_errstr);
+
         ret = 0;
 out:
+        if (ret && op_errstr && !*op_errstr)
+                gf_asprintf (op_errstr, "Enabling quota on volume %s has been "
+                             "unsuccessful", volinfo->volname);
         return ret;
 }
 
 int32_t
-glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
+glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr,
+                        gf_boolean_t *crawl)
 {
-        int32_t  ret            = -1;
-        char    *quota_status   = NULL, *quota_limits = NULL;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", op_errstr, out);
+        int32_t    ret            = -1;
+        int        i              =  0;
+        char      *quota_status   = NULL;
+        char      *value          = NULL;
+        xlator_t  *this           = NULL;
+        glusterd_conf_t *conf     = NULL;
+        char *quota_options[]     = {"features.soft-timeout",
+                                     "features.hard-timeout",
+                                     "features.alert-time",
+                                     "features.default-soft-limit", NULL};
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
 
         ret = glusterd_check_if_quota_trans_enabled (volinfo);
         if (ret == -1) {
@@ -489,271 +371,768 @@ glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
 
         quota_status = gf_strdup ("off");
         if (!quota_status) {
-                gf_log ("", GF_LOG_ERROR, "memory allocation failed");
-                *op_errstr = gf_strdup ("Disabling quota has been unsuccessful");
+                gf_log (this->name, GF_LOG_ERROR, "memory allocation failed");
+                ret = -1;
                 goto out;
         }
 
         ret = dict_set_dynstr (volinfo->dict, VKEY_FEATURES_QUOTA, quota_status);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "dict set failed");
-                *op_errstr = gf_strdup ("Disabling quota has been unsuccessful");
+                gf_log (this->name, GF_LOG_ERROR, "dict set failed");
                 goto out;
         }
 
-        *op_errstr = gf_strdup ("Disabling quota has been successful");
+        for (i = 0; quota_options [i]; i++) {
+                ret = glusterd_volinfo_get (volinfo, quota_options[i], &value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_INFO, "failed to get option"
+                                                         " %s",
+                                                         quota_options[i]);
+                } else {
+                dict_del (volinfo->dict, quota_options[i]);
+                }
+        }
+
+        //Remove aux mount of the volume on every node in the cluster
+        ret = glusterd_remove_auxiliary_mount (volinfo->volname);
+        if (ret)
+                goto out;
 
-        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_LIMIT_USAGE,
-                                    &quota_limits);
+        *crawl = _gf_true;
+
+        (void) glusterd_clean_up_quota_store (volinfo);
+
+        ret = 0;
+out:
+        if (ret && op_errstr && !*op_errstr)
+                gf_asprintf (op_errstr, "Disabling quota on volume %s has been "
+                             "unsuccessful", volinfo->volname);
+        return ret;
+}
+
+
+static int
+glusterd_set_quota_limit (char *volname, char *path, char *hard_limit,
+                          char *soft_limit, char **op_errstr)
+{
+        int               ret                = -1;
+        xlator_t         *this               = NULL;
+        char              abspath[PATH_MAX]  = {0,};
+        glusterd_conf_t  *priv               = NULL;
+        double           soft_lim            = 0;
+
+        typedef struct quota_limits {
+                int64_t hl;
+                int64_t sl;
+        } __attribute__ ((__packed__)) quota_limits_t;
+
+	quota_limits_t existing_limit = {0,};
+	quota_limits_t new_limit = {0,};
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
+        ret = gf_lstat_dir (abspath, NULL);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "failed to get the quota limits");
+                gf_asprintf (op_errstr, "Failed to find the directory %s. "
+                             "Reason : %s", abspath, strerror (errno));
+                goto out;
+        }
+
+        if (!soft_limit) {
+                ret = sys_lgetxattr (abspath,
+                                     "trusted.glusterfs.quota.limit-set",
+                                     (void *)&existing_limit,
+                                     sizeof (existing_limit));
+                if (ret < 0) {
+                        switch (errno) {
+                        case ENOATTR:
+                                existing_limit.sl = -1;
+                            break;
+                        default:
+                                gf_asprintf (op_errstr, "Failed to get the xattr "
+                                             "'trusted.glusterfs.quota.limit-set' from "
+                                             "%s. Reason : %s", abspath,
+                                             strerror (errno));
+                                goto out;
+                        }
+                } else {
+                        existing_limit.hl = ntoh64 (existing_limit.hl);
+                        existing_limit.sl = ntoh64 (existing_limit.sl);
+                }
+                new_limit.sl = existing_limit.sl;
+
         } else {
-                GF_FREE (quota_limits);
+                ret = gf_string2percent (soft_limit, &soft_lim);
+                if (ret)
+                        goto out;
+                new_limit.sl = soft_lim;
         }
 
-        dict_del (volinfo->dict, VKEY_FEATURES_LIMIT_USAGE);
+        new_limit.sl = hton64 (new_limit.sl);
+
+        ret = gf_string2bytesize (hard_limit, (uint64_t*)&new_limit.hl);
+        if (ret)
+                goto out;
+
+        new_limit.hl = hton64 (new_limit.hl);
+
+        ret = sys_lsetxattr (abspath, "trusted.glusterfs.quota.limit-set",
+                             (char *)(void *)&new_limit, sizeof (new_limit), 0);
+        if (ret) {
+                gf_asprintf (op_errstr, "setxattr of "
+                             "'trusted.glusterfs.quota.limit-set' failed on %s."
+                             " Reason : %s", abspath, strerror (errno));
+                goto out;
+        }
+        ret = 0;
 
 out:
         return ret;
 }
 
-int32_t
-glusterd_quota_limit_usage (glusterd_volinfo_t *volinfo, dict_t *dict, char **op_errstr)
+static int
+glusterd_update_quota_conf_version (glusterd_volinfo_t *volinfo)
 {
-        int32_t          ret    = -1;
-        char            *path   = NULL;
-        char            *limit  = NULL;
-        char            *value  = NULL;
-        char             msg [1024] = {0,};
-        char            *quota_limits = NULL;
+        volinfo->quota_conf_version++;
+        return 0;
+}
 
-        GF_VALIDATE_OR_GOTO ("glusterd", dict, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", op_errstr, out);
+/*The function glusterd_find_gfid_match () does the following:
+ * Given a buffer of gfids, the number of bytes read and the key gfid that needs
+ * to be found, the function compares 16 bytes at a time from @buf against
+ * @gfid.
+ *
+ * What happens when the match is found:
+ * i. If the function was called as part of 'limit-usage' operation, the call
+ *    returns with write_byte_count = bytes_read
+ *ii. If the function as called as part of 'quota remove' operation, @buf
+ *    is modified in memory such that the match is deleted from the buffer, and
+ *    also @write_byte_count is set to original buf size minus the sixteen bytes
+ *    that was deleted as part of 'remove'.
+ *
+ * What happens when the match is not found in the current buffer:
+ * The function returns with write_byte_count = bytes_read, which means to say
+ * that the caller of this function must write the entire buffer to the tmp file
+ * and continue the search.
+ */
+static gf_boolean_t
+glusterd_find_gfid_match (uuid_t gfid, unsigned char *buf, size_t bytes_read,
+                          int opcode, size_t *write_byte_count)
+{
+        int           gfid_index  = 0;
+        int           shift_count = 0;
+        unsigned char tmp_buf[17] = {0,};
+
+        while (gfid_index != bytes_read) {
+                memcpy ((void *)tmp_buf, (void *)&buf[gfid_index], 16);
+                if (!uuid_compare (gfid, tmp_buf)) {
+                        if (opcode == GF_QUOTA_OPTION_TYPE_REMOVE) {
+                                shift_count = bytes_read - (gfid_index + 16);
+                                memmove ((void *)&buf[gfid_index],
+                                         (void *)&buf[gfid_index+16],
+                                         shift_count);
+                                *write_byte_count = bytes_read - 16;
+                        } else {
+                                *write_byte_count = bytes_read;
+                        }
+                        return _gf_true;
+                } else {
+                        gfid_index+=16;
+                }
+        }
+        if (gfid_index == bytes_read)
+                *write_byte_count = bytes_read;
 
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                *op_errstr = gf_strdup ("Quota is disabled, please enable "
-                                        "quota");
+        return _gf_false;
+}
+
+/* The function glusterd_copy_to_tmp_file() reads the "remaining" bytes from
+ * the source fd and writes them to destination fd, at the rate of 128K bytes
+ * of read+write at a time.
+ */
+
+static int
+glusterd_copy_to_tmp_file (int src_fd, int dst_fd)
+{
+        int            ret         = 0;
+        size_t         entry_sz    = 131072;
+        ssize_t        bytes_read  = 0;
+        unsigned char  buf[131072] = {0,};
+        xlator_t      *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        while ((bytes_read = read (src_fd, (void *)&buf, entry_sz)) > 0) {
+                if (bytes_read % 16 != 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "quota.conf "
+                                "corrupted");
+                        ret = -1;
+                        goto out;
+                }
+                ret = write (dst_fd, (void *) buf, bytes_read);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "write into quota.conf failed. Reason : %s",
+                                strerror (errno));
+                        goto out;
+                }
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_store_quota_config (glusterd_volinfo_t *volinfo, char *path,
+                             char *gfid_str, int opcode, char **op_errstr)
+{
+        int                ret                   = -1;
+        int                fd                    = -1;
+        int                conf_fd               = -1;
+        size_t             entry_sz              = 131072;
+        ssize_t            bytes_read            = 0;
+        size_t            bytes_to_write         = 0;
+        unsigned char      buf[131072]           = {0,};
+        uuid_t             gfid                  = {0,};
+        xlator_t          *this                  = NULL;
+        gf_boolean_t       found                 = _gf_false;
+        gf_boolean_t       modified              = _gf_false;
+        gf_boolean_t       is_file_empty         = _gf_false;
+        gf_boolean_t       is_first_read         = _gf_true;
+        glusterd_conf_t   *conf                  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        glusterd_store_create_quota_conf_sh_on_absence (volinfo);
+
+        fd = gf_store_mkstemp (volinfo->quota_conf_shandle);
+        if (fd < 0) {
+                ret = -1;
                 goto out;
         }
 
-        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_LIMIT_USAGE,
-                                    &quota_limits);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "failed to get the quota limits");
-                *op_errstr = gf_strdup ("failed to set limit");
+        conf_fd = open (volinfo->quota_conf_shandle->path, O_RDONLY);
+        if (conf_fd == -1) {
+                ret = -1;
                 goto out;
         }
 
-        ret = dict_get_str (dict, "path", &path);
+        ret = glusterd_store_quota_conf_skip_header (this, conf_fd);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch quota limits" );
-                *op_errstr = gf_strdup ("failed to set limit");
                 goto out;
         }
 
-        ret = dict_get_str (dict, "limit", &limit);
+        ret = glusterd_store_quota_conf_stamp_header (this, fd);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch quota limits" );
-                *op_errstr = gf_strdup ("failed to set limit");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to add header to tmp "
+                        "file.");
                 goto out;
         }
 
-        if (quota_limits) {
-                ret = _glusterd_quota_remove_limits (&quota_limits, path, NULL);
-                if (ret == -1) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                        *op_errstr = gf_strdup ("failed to set limit");
+        /* Just create empty quota.conf file if create */
+        if (GF_QUOTA_OPTION_TYPE_ENABLE == opcode) {
+                modified = _gf_true;
+                goto out;
+        }
+
+        /* Check if gfid_str is given for opts other than ENABLE */
+        if (!gfid_str) {
+                ret = -1;
+                goto out;
+        }
+        uuid_parse (gfid_str, gfid);
+
+        for (;;) {
+                bytes_read = read (conf_fd, (void*)&buf, entry_sz);
+                if (bytes_read <= 0) {
+                        /*The flag @is_first_read is TRUE when the loop is
+                         * entered, and is set to false if the first read
+                         * reads non-zero bytes of data. The flag is used to
+                         * detect if quota.conf is an empty file, but for the
+                         * header. This is done to log appropriate error message
+                         * when 'quota remove' is attempted when there are no
+                         * limits set on the given volume.
+                         */
+                        if (is_first_read)
+                                is_file_empty = _gf_true;
+                        break;
+                }
+                if ((bytes_read % 16) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "quota.conf "
+                                "corrupted");
+                        ret = -1;
                         goto out;
                 }
-        }
+                found = glusterd_find_gfid_match (gfid, buf, bytes_read, opcode,
+                                                  &bytes_to_write);
 
-        if (quota_limits == NULL) {
-                ret = gf_asprintf (&value, "%s:%s", path, limit);
+                ret = write (fd, (void *) buf, bytes_to_write);
                 if (ret == -1) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                        *op_errstr = gf_strdup ("failed to set limit");
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "write into quota.conf failed. Reason : %s",
+                                strerror (errno));
                         goto out;
                 }
-        } else {
-                ret = gf_asprintf (&value, "%s,%s:%s",
-                                   quota_limits, path, limit);
-                if (ret == -1) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                        *op_errstr = gf_strdup ("failed to set limit");
+
+                /*If the match is found in this iteration, copy the rest of
+                 * quota.conf into quota.conf.tmp and break.
+                 * Else continue with the search.
+                 */
+                if (found) {
+                        ret = glusterd_copy_to_tmp_file (conf_fd, fd);
+                        if (ret)
+                                goto out;
+                        break;
+                }
+                is_first_read = _gf_false;
+        }
+
+        switch (opcode) {
+                case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                        if (!found) {
+                                ret = write (fd, gfid, 16);
+                                if (ret == -1) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "write into quota.conf failed. "
+                                                "Reason : %s",
+                                                strerror (errno));
+                                        goto out;
+                                }
+                                modified = _gf_true;
+                        }
+                        break;
+
+                case GF_QUOTA_OPTION_TYPE_REMOVE:
+                        if (is_file_empty) {
+                                gf_asprintf (op_errstr, "Cannot remove limit on"
+                                             " %s. The quota configuration file"
+                                             " for volume %s is empty.", path,
+                                             volinfo->volname);
+                                ret = -1;
+                                goto out;
+                        } else {
+                                if (!found) {
+                                        gf_asprintf (op_errstr, "Error. gfid %s"
+                                                     " for path %s not found in"
+                                                     " store", gfid_str, path);
+                                        ret = -1;
+                                        goto out;
+                                } else {
+                                        modified = _gf_true;
+                                }
+                        }
+                        break;
+
+                default:
+                        ret = 0;
+                        break;
+        }
+
+        if (modified)
+                glusterd_update_quota_conf_version (volinfo);
+
+        ret = 0;
+out:
+        if (conf_fd != -1) {
+                close (conf_fd);
+        }
+
+        if (fd != -1) {
+                close (fd);
+        }
+
+        if (ret && (fd > 0)) {
+                gf_store_unlink_tmppath (volinfo->quota_conf_shandle);
+        } else if (!ret) {
+                ret = gf_store_rename_tmppath (volinfo->quota_conf_shandle);
+                if (modified) {
+                        ret = glusterd_compute_cksum (volinfo, _gf_true);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "compute cksum for quota conf file");
+                                goto out;
+                        }
+
+                        ret = glusterd_store_save_quota_version_and_cksum
+                                                                      (volinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "store quota version and cksum");
+                                goto out;
+                        }
+                }
+        }
+
+        return ret;
+}
+
+int32_t
+glusterd_quota_limit_usage (glusterd_volinfo_t *volinfo, dict_t *dict,
+                            int opcode, char **op_errstr)
+{
+        int32_t          ret                = -1;
+        char            *path               = NULL;
+        char            *hard_limit         = NULL;
+        char            *soft_limit         = NULL;
+        char            *gfid_str           = NULL;
+        xlator_t        *this               = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
+
+        ret = glusterd_check_if_quota_trans_enabled (volinfo);
+        if (ret == -1) {
+                *op_errstr = gf_strdup ("Quota is disabled, please enable "
+                                        "quota");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "path", &path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch path");
+                goto out;
+        }
+        ret = gf_canonicalize_path (path);
+        if (ret)
+                goto out;
+
+        ret = dict_get_str (dict, "hard-limit", &hard_limit);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch hard limit");
+                goto out;
+        }
+
+        if (dict_get (dict, "soft-limit")) {
+                ret = dict_get_str (dict, "soft-limit", &soft_limit);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
+                                "soft limit");
                         goto out;
                 }
+        }
 
-                GF_FREE (quota_limits);
+        if (is_origin_glusterd (dict)) {
+                ret = glusterd_set_quota_limit (volinfo->volname, path,
+                                                hard_limit, soft_limit,
+                                                op_errstr);
+                if (ret)
+                        goto out;
         }
 
-        quota_limits = value;
+        ret = dict_get_str (dict, "gfid", &gfid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get gfid of path "
+                        "%s", path);
+                goto out;
+        }
+
+        ret = glusterd_store_quota_config (volinfo, path, gfid_str, opcode,
+                                           op_errstr);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+
+        if (ret && op_errstr && !*op_errstr)
+                gf_asprintf (op_errstr, "Failed to set hard limit on path %s "
+                             "for volume %s", path, volinfo->volname);
+        return ret;
+}
 
-        ret = dict_set_str (volinfo->dict, VKEY_FEATURES_LIMIT_USAGE,
-                            quota_limits);
+static int
+glusterd_remove_quota_limit (char *volname, char *path, char **op_errstr)
+{
+        int               ret                = -1;
+        xlator_t         *this               = NULL;
+        char              abspath[PATH_MAX]  = {0,};
+        glusterd_conf_t  *priv               = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
+        ret = gf_lstat_dir (abspath, NULL);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set quota limits" );
-                *op_errstr = gf_strdup ("failed to set limit");
+                gf_asprintf (op_errstr, "Failed to find the directory %s. "
+                             "Reason : %s", abspath, strerror (errno));
                 goto out;
         }
-        snprintf (msg, 1024, "limit set on %s", path);
-        *op_errstr = gf_strdup (msg);
 
+        ret = sys_lremovexattr (abspath, "trusted.glusterfs.quota.limit-set");
+        if (ret) {
+                gf_asprintf (op_errstr, "removexattr failed on %s. Reason : %s",
+                             abspath, strerror (errno));
+                goto out;
+        }
         ret = 0;
+
 out:
         return ret;
 }
 
 int32_t
-glusterd_quota_remove_limits (glusterd_volinfo_t *volinfo, dict_t *dict, char **op_errstr)
+glusterd_quota_remove_limits (glusterd_volinfo_t *volinfo, dict_t *dict,
+                              int opcode, char **op_errstr)
 {
         int32_t         ret                   = -1;
-        char            str [PATH_MAX + 1024] = {0,};
-        char            *quota_limits         = NULL;
         char            *path                 = NULL;
-        gf_boolean_t     flag                 = _gf_false;
+        char            *gfid_str             = NULL;
+        xlator_t        *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
-        GF_VALIDATE_OR_GOTO ("glusterd", dict, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", op_errstr, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
 
         ret = glusterd_check_if_quota_trans_enabled (volinfo);
         if (ret == -1) {
-                *op_errstr = gf_strdup ("Quota is disabled, please enable quota");
+                *op_errstr = gf_strdup ("Quota is disabled, please enable "
+                                        "quota");
                 goto out;
         }
 
-        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_LIMIT_USAGE,
-                                    &quota_limits);
+        ret = dict_get_str (dict, "path", &path);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "failed to get the quota limits");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch path");
                 goto out;
         }
 
-        ret = dict_get_str (dict, "path", &path);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch quota limits" );
+        ret = gf_canonicalize_path (path);
+        if (ret)
                 goto out;
+
+        if (is_origin_glusterd (dict)) {
+                ret = glusterd_remove_quota_limit (volinfo->volname, path,
+                                                   op_errstr);
+                if (ret)
+                        goto out;
         }
 
-        ret = _glusterd_quota_remove_limits (&quota_limits, path, &flag);
-        if (ret == -1) {
-                if (flag == _gf_true)
-                        snprintf (str, sizeof (str), "Removing limit on %s has "
-                                  "been unsuccessful", path);
-                else
-                        snprintf (str, sizeof (str), "%s has no limit set", path);
-                *op_errstr = gf_strdup (str);
+        ret = dict_get_str (dict, "gfid", &gfid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get gfid of path "
+                        "%s", path);
                 goto out;
-        } else {
-                if (flag == _gf_true)
-                        snprintf (str, sizeof (str), "Removed quota limit on "
-                                  "%s", path);
-                else
-                        snprintf (str, sizeof (str), "no limit set on %s",
-                                  path);
-                *op_errstr = gf_strdup (str);
-        }
-
-        if (quota_limits) {
-                ret = dict_set_str (volinfo->dict, VKEY_FEATURES_LIMIT_USAGE,
-                                    quota_limits);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to set quota limits" );
-                        goto out;
-                }
-        } else {
-               dict_del (volinfo->dict, VKEY_FEATURES_LIMIT_USAGE);
         }
 
+        ret = glusterd_store_quota_config (volinfo, path, gfid_str, opcode,
+                                           op_errstr);
+        if (ret)
+                goto out;
+
+
         ret = 0;
 
 out:
         return ret;
 }
 
+int
+glusterd_set_quota_option (glusterd_volinfo_t *volinfo, dict_t *dict,
+                           char *key, char **op_errstr)
+{
+        int        ret    = 0;
+        char      *value  = NULL;
+        xlator_t  *this   = NULL;
+        char      *option = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_check_if_quota_trans_enabled (volinfo);
+        if (ret == -1) {
+                gf_asprintf (op_errstr, "Cannot set %s. Quota on volume %s is "
+                                        "disabled", key, volinfo->volname);
+                return -1;
+        }
+
+        ret = dict_get_str (dict, "value", &value);
+        if(ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Option value absent.");
+                return -1;
+        }
+
+        option = gf_strdup (value);
+        ret = dict_set_dynstr (volinfo->dict, key, option);
+        if(ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set option %s",
+                        key);
+                return -1;
+        }
+
+        return 0;
+}
+
+static int
+glusterd_quotad_op (int opcode)
+{
+        int ret = -1;
+
+        switch (opcode) {
+                case GF_QUOTA_OPTION_TYPE_ENABLE:
+                case GF_QUOTA_OPTION_TYPE_DISABLE:
+
+                        if (glusterd_all_volumes_with_quota_stopped ())
+                                ret = glusterd_quotad_stop ();
+                        else
+                                ret = glusterd_check_generate_start_quotad ();
+                        break;
+
+                default:
+                        ret = 0;
+                        break;
+        }
+        return ret;
+}
 
 int
-glusterd_op_quota (dict_t *dict, char **op_errstr)
+glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
         glusterd_volinfo_t     *volinfo      = NULL;
         int32_t                 ret          = -1;
         char                   *volname      = NULL;
-        dict_t                 *ctx          = NULL;
         int                     type         = -1;
         gf_boolean_t            start_crawl  = _gf_false;
         glusterd_conf_t        *priv         = NULL;
+        xlator_t               *this         = NULL;
 
         GF_ASSERT (dict);
         GF_ASSERT (op_errstr);
 
-        priv = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name " );
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "type", &type);
 
-        if (type == GF_QUOTA_OPTION_TYPE_ENABLE) {
-                ret = glusterd_quota_enable (volinfo, op_errstr, &start_crawl);
-                if (ret < 0)
-                        goto out;
-
-                goto create_vol;
+        if ((priv->op_version == GD_OP_VERSION_MIN) &&
+            (type > GF_QUOTA_OPTION_TYPE_VERSION)) {
+                gf_asprintf (op_errstr, "Volume quota failed. The cluster is "
+                                        "operating at version %d. Quota command"
+                                        " %s is unavailable in this version.",
+                                        priv->op_version,
+                                        gd_quota_op_list[type]);
+                ret = -1;
+                goto out;
         }
 
-        if (type == GF_QUOTA_OPTION_TYPE_DISABLE) {
-                ret = glusterd_quota_disable (volinfo, op_errstr);
-                if (ret < 0)
-                        goto out;
+        switch (type) {
+                case GF_QUOTA_OPTION_TYPE_ENABLE:
+                        ret = glusterd_quota_enable (volinfo, op_errstr,
+                                                     &start_crawl);
+                        if (ret < 0)
+                                goto out;
+                        break;
 
-                goto create_vol;
-        }
+                case GF_QUOTA_OPTION_TYPE_DISABLE:
+                        ret = glusterd_quota_disable (volinfo, op_errstr,
+                                                      &start_crawl);
+                        if (ret < 0)
+                                goto out;
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
-                ret = glusterd_quota_limit_usage (volinfo, dict, op_errstr);
-                if (ret < 0)
+                        break;
+
+                case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                        ret = glusterd_quota_limit_usage (volinfo, dict, type,
+                                                          op_errstr);
                         goto out;
 
-                goto create_vol;
-        }
+                case GF_QUOTA_OPTION_TYPE_REMOVE:
+                        ret = glusterd_quota_remove_limits (volinfo, dict, type,
+                                                            op_errstr);
+                        goto out;
 
-        if (type == GF_QUOTA_OPTION_TYPE_REMOVE) {
-                ret = glusterd_quota_remove_limits (volinfo, dict, op_errstr);
-                if (ret < 0)
+                case GF_QUOTA_OPTION_TYPE_LIST:
+                        ret = glusterd_check_if_quota_trans_enabled (volinfo);
+                        if (ret == -1) {
+                                *op_errstr = gf_strdup ("Cannot list limits, "
+                                                        "quota is disabled");
+                                goto out;
+                        }
+                        ret = glusterd_quota_get_default_soft_limit (volinfo,
+                                                               rsp_dict);
                         goto out;
 
-                goto create_vol;
-        }
+                case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+                        ret = glusterd_set_quota_option (volinfo, dict,
+                                                        "features.soft-timeout",
+                                                         op_errstr);
+                        if (ret)
+                                goto out;
+                        break;
 
-        if (type == GF_QUOTA_OPTION_TYPE_LIST) {
-                ret = glusterd_check_if_quota_trans_enabled (volinfo);
-                if (ret == -1) {
-                        *op_errstr = gf_strdup ("cannot list the limits, "
-                                                "quota is disabled");
-                        goto out;
-                }
+                case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+                        ret = glusterd_set_quota_option (volinfo, dict,
+                                                        "features.hard-timeout",
+                                                         op_errstr);
+                        if (ret)
+                                goto out;
+                        break;
 
-                ret = glusterd_quota_get_limit_usages (priv, volinfo, volname,
-                                                       dict, op_errstr);
+                case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+                        ret = glusterd_set_quota_option (volinfo, dict,
+                                                         "features.alert-time",
+                                                         op_errstr);
+                        if (ret)
+                                goto out;
+                        break;
 
-                goto out;
+                case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+                        ret = glusterd_set_quota_option (volinfo, dict,
+                                                  "features.default-soft-limit",
+                                                  op_errstr);
+                        if (ret)
+                                goto out;
+                        break;
+
+                default:
+                        gf_asprintf (op_errstr, "Quota command failed. Invalid "
+                                     "opcode");
+                        ret = -1;
+                        goto out;
+        }
+
+        if (priv->op_version > GD_OP_VERSION_MIN) {
+                ret = glusterd_quotad_op (type);
+                if (ret)
+                        goto out;
         }
-create_vol:
+
         ret = glusterd_create_volfiles_and_notify_services (volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to re-create volfile for"
-                                          " 'quota'");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to re-create "
+                                                  "volfiles");
                 ret = -1;
                 goto out;
         }
@@ -762,82 +1141,311 @@ create_vol:
         if (ret)
                 goto out;
 
-        if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                ret = glusterd_check_generate_start_nfs ();
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                if (priv->op_version == GD_OP_VERSION_MIN)
+                        ret = glusterd_check_generate_start_nfs ();
+        }
+
+        if (rsp_dict && start_crawl == _gf_true)
+                glusterd_quota_initiate_fs_crawl (priv, volname, type);
 
         ret = 0;
-
 out:
-        ctx = glusterd_op_get_ctx ();
-        if (ctx && start_crawl == _gf_true)
-                glusterd_quota_initiate_fs_crawl (priv, volname);
+        return ret;
+}
+
+/*
+ * glusterd_get_gfid_from_brick() fetches the 'trusted.gfid' attribute of @path
+ * from each brick in the backend and places the same in the rsp_dict with the
+ * keys being gfid0, gfid1, gfid2 and so on. The absence of @path in the backend
+ * is not treated as error.
+ */
+static int
+glusterd_get_gfid_from_brick (dict_t *dict, glusterd_volinfo_t *volinfo,
+                              dict_t *rsp_dict, char **op_errstr)
+{
+        int                    ret                    = -1;
+        int                    count                  = 0;
+        char                  *path                   = NULL;
+        char                   backend_path[PATH_MAX] = {0,};
+        xlator_t              *this                   = NULL;
+        glusterd_conf_t       *priv                   = NULL;
+        glusterd_brickinfo_t  *brickinfo              = NULL;
+        char                   key[256]               = {0,};
+        char                  *gfid_str               = NULL;
+        uuid_t                 gfid;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "path", &path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get path");
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                if (brickinfo->vg[0])
+                        continue;
+
+                snprintf (backend_path, sizeof (backend_path), "%s%s",
+                          brickinfo->path, path);
+
+                ret = gf_lstat_dir (backend_path, NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_INFO, "Failed to find "
+                                "directory %s. Reason : %s", backend_path,
+                                strerror (errno));
+                        ret = 0;
+                        continue;
+                }
+                ret = sys_lgetxattr (backend_path, GFID_XATTR_KEY, gfid, 16);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO, "Failed to get "
+                                "extended attribute %s for directory %s. "
+                                "Reason : %s", GFID_XATTR_KEY, backend_path,
+                                     strerror (errno));
+                        ret = 0;
+                        continue;
+                }
+                snprintf (key, sizeof (key), "gfid%d", count);
+
+                gfid_str = gf_strdup (uuid_utoa (gfid));
+                if (!gfid_str) {
+                        ret = -1;
+                        goto out;
+                }
 
-        if (ctx && *op_errstr) {
-                ret = dict_set_dynstr (ctx, "errstr", *op_errstr);
+                ret = dict_set_dynstr (rsp_dict, key, gfid_str);
                 if (ret) {
-                        GF_FREE (*op_errstr);
-                        gf_log ("", GF_LOG_DEBUG,
-                                "failed to set error message in ctx");
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to place "
+                                "gfid of %s in dict", backend_path);
+                        GF_FREE (gfid_str);
+                        goto out;
                 }
-                *op_errstr = NULL;
+                count++;
         }
 
+        ret = dict_set_int32 (rsp_dict, "count", count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set count");
+                goto out;
+        }
+
+        ret = 0;
+out:
         return ret;
 }
 
+static int
+_glusterd_validate_quota_opts (dict_t *dict, int type, char **errstr)
+{
+        int                     ret = -1;
+        xlator_t                *this = THIS;
+        void                    *quota_xl = NULL;
+        volume_opt_list_t       opt_list = {{0},};
+        volume_option_t         *opt = NULL;
+        char                    *key = NULL;
+        char                    *value = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (this);
+
+        ret = xlator_volopt_dynload ("features/quota", &quota_xl, &opt_list);
+        if (ret)
+                goto out;
+
+        switch (type) {
+        case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+        case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+        case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+        case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+                key = (char *)gd_quota_op_list[type];
+                break;
+        default:
+                ret = -1;
+                goto out;
+        }
+
+        opt = xlator_volume_option_get_list (&opt_list, key);
+        if (!opt) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Unknown option: %s", key);
+                goto out;
+        }
+        ret = dict_get_str (dict, "value", &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Value not found for key %s",
+                        key);
+                goto out;
+        }
+
+        ret = xlator_option_validate (this, key, value, opt, errstr);
+
+out:
+        if (quota_xl) {
+                dlclose (quota_xl);
+                quota_xl = NULL;
+        }
+        return ret;
+}
 int
-glusterd_op_stage_quota (dict_t *dict, char **op_errstr)
+glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                ret           = 0;
-        char              *volname       = NULL;
-        gf_boolean_t       exists        = _gf_false;
-        int                type          = 0;
-        dict_t            *ctx           = NULL;
+        int                 ret            = 0;
+        char               *volname        = NULL;
+        gf_boolean_t        exists         = _gf_false;
+        int                 type           = 0;
+        xlator_t           *this           = NULL;
+        glusterd_conf_t    *priv           = NULL;
+        glusterd_volinfo_t *volinfo        = NULL;
+        char               *hard_limit_str = NULL;
+        uint64_t           hard_limit      = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
 
         GF_ASSERT (dict);
         GF_ASSERT (op_errstr);
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         exists = glusterd_check_volume_exists (volname);
         if (!exists) {
-                gf_log ("", GF_LOG_ERROR, "Volume with name: %s "
-                                "does not exist",
-                                volname);
-                *op_errstr = gf_strdup ("Invalid volume name");
+                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+                ret = -1;
+                goto out;
+        }
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+                goto out;
+        }
+
+        if (!glusterd_is_volume_started (volinfo)) {
+                *op_errstr = gf_strdup ("Volume is stopped, start volume "
+                                        "before executing quota command.");
                 ret = -1;
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "type", &type);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get 'type' for quota op");
-                *op_errstr = gf_strdup ("Volume quota failed, internal error "
-                                        ", unable to get type of operation");
+                *op_errstr = gf_strdup ("Volume quota failed, internal error, "
+                                        "unable to get type of operation");
+                goto out;
+        }
+
+        if ((!glusterd_is_volume_quota_enabled (volinfo)) &&
+            (type != GF_QUOTA_OPTION_TYPE_ENABLE)) {
+                *op_errstr = gf_strdup ("Quota is disabled, please enable "
+                                        "quota");
+                ret = -1;
                 goto out;
         }
 
+        if ((priv->op_version == GD_OP_VERSION_MIN) &&
+            (type > GF_QUOTA_OPTION_TYPE_VERSION)) {
+                gf_asprintf (op_errstr, "Volume quota failed. The cluster is "
+                                        "operating at version %d. Quota command"
+                                        " %s is unavailable in this version.",
+                                         priv->op_version,
+                                         gd_quota_op_list[type]);
+                ret = -1;
+                goto out;
+        }
+
+        if ((GF_QUOTA_OPTION_TYPE_ENABLE != type) &&
+            (glusterd_check_if_quota_trans_enabled (volinfo) != 0)) {
+                ret = -1;
+                gf_asprintf (op_errstr, "Quota is not enabled on volume %s",
+                             volname);
+                goto out;
+        }
 
-        ctx = glusterd_op_get_ctx();
-        if (ctx && (type == GF_QUOTA_OPTION_TYPE_ENABLE
-                    || type == GF_QUOTA_OPTION_TYPE_LIST)) {
+        switch (type) {
+        case GF_QUOTA_OPTION_TYPE_ENABLE:
+        case GF_QUOTA_OPTION_TYPE_LIST:
                 /* Fuse mount req. only for enable & list-usage options*/
-                if (!glusterd_is_fuse_available ()) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to open /dev/"
-                                "fuse (%s), quota command failed",
-                                strerror (errno));
+                if (is_origin_glusterd (dict) &&
+                    !glusterd_is_fuse_available ()) {
                         *op_errstr = gf_strdup ("Fuse unavailable");
                         ret = -1;
                         goto out;
                 }
+                break;
+
+        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                ret = dict_get_str (dict, "hard-limit", &hard_limit_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Faild to get hard-limit from dict");
+                        goto out;
+                }
+                ret = gf_string2bytesize (hard_limit_str, &hard_limit);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to convert hard-limit string to value");
+                        goto out;
+                }
+                if (hard_limit > INT64_MAX) {
+                        ret = -1;
+                        ret = gf_asprintf (op_errstr, "Hard-limit %s is greater"
+                                           " than %"PRId64"bytes. Please set a "
+                                           "smaller limit.", hard_limit_str,
+                                           INT64_MAX);
+                        gf_log (this->name, GF_LOG_ERROR, "hard-limit %s "
+                                "greater than INT64_MAX", hard_limit_str);
+                        goto out;
+                }
+                /*The break statement is missing here to allow intentional fall
+                 * through of code execution to the next switch case
+                 */
+
+        case GF_QUOTA_OPTION_TYPE_REMOVE:
+                ret = glusterd_get_gfid_from_brick (dict, volinfo, rsp_dict,
+                                                    op_errstr);
+                if (ret)
+                        goto out;
+                break;
+
+        case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+        case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+        case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+        case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+                ret = _glusterd_validate_quota_opts (dict, type, op_errstr);
+                if (ret)
+                        goto out;
+                break;
+
+        default:
+                break;
         }
 
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        ret = 0;
 
-        return ret;
+ out:
+        if (ret && op_errstr && *op_errstr)
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 936a3b26e..bdedf4c04 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -41,99 +31,38 @@
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
 #include "run.h"
+#include "glusterd-volgen.h"
 
 #include "syscall.h"
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 
 int32_t
-glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe);
-
 int
-glusterd_defrag_update_state (glusterd_volinfo_t *volinfo,
-                              glusterd_defrag_info_t *defrag)
+glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
+                                size_t len, glusterd_op_t op)
 {
-        int     ret             = -1;
-        int     cmd             = 0;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (defrag);
-
-        if (volinfo->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+        int      ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        /* Check only if operation is not remove-brick */
+        if ((GD_OP_REMOVE_BRICK != op) &&
+            !gd_is_remove_brick_committed (volinfo)) {
+                gf_log (this->name, GF_LOG_DEBUG, "A remove-brick task on "
+                        "volume %s is not yet committed", volinfo->volname);
+                snprintf (op_errstr, len, "A remove-brick task on volume %s is"
+                          " not yet committed. Either commit or stop the "
+                          "remove-brick task.", volinfo->volname);
                 goto out;
         }
 
-        LOCK (&defrag->lock);
-        {
-                cmd = defrag->cmd;
-                if ((cmd == GF_DEFRAG_CMD_START) || (cmd ==
-                        GF_DEFRAG_CMD_START_FORCE) || (cmd ==
-                        GF_DEFRAG_CMD_START_LAYOUT_FIX))
-                        volinfo->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
-                else if (cmd == GF_DEFRAG_CMD_STOP)
-                        volinfo->defrag_status = GF_DEFRAG_STATUS_STOPPED;
-        }
-        UNLOCK (&defrag->lock);
-
-        ret = 0;
-out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-void
-glusterd_rebalance_cmd_attempted_log (int cmd, char *volname)
-{
-        switch (cmd) {
-                case GF_DEFRAG_CMD_START_LAYOUT_FIX:
-                        gf_cmd_log ("Volume rebalance"," on volname: %s "
-                                    "cmd: start fix layout , attempted",
-                                    volname);
-                        gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
-                                "volume start layout fix on %s", volname);
-                        break;
-                case GF_DEFRAG_CMD_START_FORCE:
-                        gf_cmd_log ("Volume rebalance"," on volname: %s "
-                                    "cmd: start data force attempted",
-                                    volname);
-                        gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
-                                "volume start migrate data on %s", volname);
-                        break;
-                case GF_DEFRAG_CMD_START:
-                        gf_cmd_log ("Volume rebalance"," on volname: %s "
-                                    "cmd: start, attempted", volname);
-                        gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
-                                "volume start on %s", volname);
-                        break;
-                case GF_DEFRAG_CMD_STOP:
-                        gf_cmd_log ("Volume rebalance"," on volname: %s "
-                                    "cmd: stop, attempted", volname);
-                        gf_log ("glusterd", GF_LOG_INFO, "Received rebalance "
-                                "volume stop on %s", volname);
-                        break;
-                default:
-                        break;
-        }
-}
-
-void
-glusterd_rebalance_cmd_log (int cmd, char *volname, int status)
-{
-        if (cmd != GF_DEFRAG_CMD_STATUS) {
-                gf_cmd_log ("volume rebalance"," on volname: %s %d %s",
-                            volname, cmd, ((status)?"FAILED":"SUCCESS"));
-        }
-}
-
-int
-glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
-                                size_t len)
-{
-        int     ret = -1;
-
         if (glusterd_is_defrag_on (volinfo)) {
-                gf_log ("glusterd", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "rebalance on volume %s already started",
                         volinfo->volname);
                 snprintf (op_errstr, len, "Rebalance on %s is already started",
@@ -143,7 +72,7 @@ glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
 
         if (glusterd_is_rb_started (volinfo) ||
             glusterd_is_rb_paused (volinfo)) {
-                gf_log ("glusterd", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Rebalance failed as replace brick is in progress on volume %s",
                         volinfo->volname);
                 snprintf (op_errstr, len, "Rebalance failed as replace brick is in progress on "
@@ -152,13 +81,14 @@ glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
         }
         ret = 0;
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+
 int32_t
-glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
-                        rpc_clnt_event_t event, void *data)
+__glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
 {
         glusterd_volinfo_t      *volinfo = NULL;
         glusterd_defrag_info_t  *defrag  = NULL;
@@ -174,12 +104,12 @@ glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
         if (!volinfo)
                 return 0;
 
-        defrag = volinfo->defrag;
+        defrag = volinfo->rebal.defrag;
         if (!defrag)
                 return 0;
 
         if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
-                volinfo->defrag = NULL;
+                volinfo->rebal.defrag = NULL;
 
         GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
 
@@ -196,7 +126,7 @@ glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
                 UNLOCK (&defrag->lock);
 
                gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_CONNECT",
-                        rpc->conn.trans->name);
+                        rpc->conn.name);
                break;
         }
 
@@ -211,32 +141,32 @@ glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
                 }
                 UNLOCK (&defrag->lock);
 
-                if (!glusterd_is_service_running (pidfile, NULL)) {
-                        glusterd_defrag_update_state (volinfo, defrag);
-                } else {
-                        volinfo->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                }
-
-                /* Success or failure, Reset cmd in volinfo */
-
-                volinfo->defrag_cmd = 0;
+                if (!gf_is_service_running (pidfile, NULL)) {
+                        if (volinfo->rebal.defrag_status ==
+                                                GF_DEFRAG_STATUS_STARTED) {
+                                volinfo->rebal.defrag_status =
+                                                   GF_DEFRAG_STATUS_FAILED;
+                        }
+                 }
 
-                glusterd_store_volinfo (volinfo,
-                                        GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                glusterd_store_perform_node_state_store (volinfo);
 
                 if (defrag->rpc) {
-                        rpc_clnt_unref (defrag->rpc);
+                        glusterd_rpc_clnt_unref (priv, defrag->rpc);
                         defrag->rpc = NULL;
                 }
                 if (defrag->cbk_fn)
-                        defrag->cbk_fn (volinfo, volinfo->defrag_status);
+                        defrag->cbk_fn (volinfo,
+                                        volinfo->rebal.defrag_status);
 
-                if (defrag)
-                        GF_FREE (defrag);
+                GF_FREE (defrag);
                 gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_DISCONNECT",
-                        rpc->conn.trans->name);
+                        rpc->conn.name);
                 break;
         }
+        case RPC_CLNT_DESTROY:
+                glusterd_volinfo_unref (volinfo);
+                break;
         default:
                 gf_log ("", GF_LOG_TRACE,
                         "got some other RPC event %d", event);
@@ -247,113 +177,203 @@ glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
         return ret;
 }
 
+int32_t
+glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+                        rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event,
+                                           data, __glusterd_defrag_notify);
+}
+
 int
 glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
-                              size_t len, int cmd, defrag_cbk_fn_t cbk)
+                              size_t len, int cmd, defrag_cbk_fn_t cbk,
+                              glusterd_op_t op)
 {
         int                    ret = -1;
         glusterd_defrag_info_t *defrag =  NULL;
         runner_t               runner = {0,};
         glusterd_conf_t        *priv = NULL;
         char                   defrag_path[PATH_MAX];
-        struct stat            buf = {0,};
         char                   sockfile[PATH_MAX] = {0,};
         char                   pidfile[PATH_MAX] = {0,};
         char                   logfile[PATH_MAX] = {0,};
-        dict_t                 *options = NULL;
+        char                   valgrind_logfile[PATH_MAX] = {0,};
 
         priv    = THIS->private;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (op_errstr);
 
-        ret = glusterd_defrag_start_validate (volinfo, op_errstr, len);
+        ret = glusterd_defrag_start_validate (volinfo, op_errstr, len, op);
         if (ret)
                 goto out;
-        if (!volinfo->defrag)
-                volinfo->defrag = GF_CALLOC (1, sizeof (glusterd_defrag_info_t),
-                                             gf_gld_mt_defrag_info);
-        if (!volinfo->defrag)
+        if (!volinfo->rebal.defrag)
+                volinfo->rebal.defrag =
+                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+                                   gf_gld_mt_defrag_info);
+        if (!volinfo->rebal.defrag)
                 goto out;
 
-        defrag = volinfo->defrag;
+        defrag = volinfo->rebal.defrag;
 
         defrag->cmd = cmd;
 
-        LOCK_INIT (&defrag->lock);
+        volinfo->rebal.defrag_cmd = cmd;
+        volinfo->rebal.op = op;
 
-        volinfo->defrag_status = GF_DEFRAG_STATUS_STARTED;
+        LOCK_INIT (&defrag->lock);
 
-        volinfo->rebalance_files = 0;
-        volinfo->rebalance_data = 0;
-        volinfo->lookedup_files = 0;
+        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
 
-        volinfo->defrag_cmd = cmd;
-        glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        glusterd_volinfo_reset_defrag_stats (volinfo);
+        glusterd_store_perform_node_state_store (volinfo);
 
         GLUSTERD_GET_DEFRAG_DIR (defrag_path, volinfo, priv);
-        ret = stat (defrag_path, &buf);
-        if (ret && (errno == ENOENT)) {
-                runinit (&runner);
-                runner_add_args (&runner, "mkdir", "-p", defrag_path, NULL);
-                ret = runner_run_reuse (&runner);
-                if (ret) {
-                        runner_log (&runner, "glusterd", GF_LOG_DEBUG,
-                                    "command failed");
-                        runner_end (&runner);
-                        goto out;
-                }
-                runner_end (&runner);
+        ret = mkdir_p (defrag_path, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create "
+                        "directory %s", defrag_path);
+                goto out;
         }
 
-        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo, priv);
+        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
         GLUSTERD_GET_DEFRAG_PID_FILE (pidfile, volinfo, priv);
         snprintf (logfile, PATH_MAX, "%s/%s-rebalance.log",
                     DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname);
         runinit (&runner);
+
+        if (priv->valgrind) {
+                snprintf (valgrind_logfile, PATH_MAX,
+                          "%s/valgrind-%s-rebalance.log",
+                          DEFAULT_LOG_FILE_DIRECTORY,
+                          volinfo->volname);
+
+                runner_add_args (&runner, "valgrind", "--leak-check=full",
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
+                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+        }
+
         runner_add_args (&runner, SBIN_DIR"/glusterfs",
                          "-s", "localhost", "--volfile-id", volinfo->volname,
                          "--xlator-option", "*dht.use-readdirp=yes",
                          "--xlator-option", "*dht.lookup-unhashed=yes",
                          "--xlator-option", "*dht.assert-no-child-down=yes",
+                         "--xlator-option", "*replicate*.data-self-heal=off",
+                         "--xlator-option",
+                         "*replicate*.metadata-self-heal=off",
+                         "--xlator-option", "*replicate*.entry-self-heal=off",
+                         "--xlator-option", "*replicate*.readdir-failover=off",
+                         "--xlator-option", "*dht.readdir-optimize=on",
                          NULL);
         runner_add_arg (&runner, "--xlator-option");
         runner_argprintf ( &runner, "*dht.rebalance-cmd=%d",cmd);
         runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf (&runner, "*dht.node-uuid=%s", uuid_utoa(priv->uuid));
+        runner_argprintf (&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
         runner_add_arg (&runner, "--socket-file");
         runner_argprintf (&runner, "%s",sockfile);
         runner_add_arg (&runner, "--pid-file");
         runner_argprintf (&runner, "%s",pidfile);
         runner_add_arg (&runner, "-l");
         runner_argprintf (&runner, logfile);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
 
-        ret = runner_run_reuse (&runner);
+        ret = runner_run_nowait (&runner);
         if (ret) {
-                runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
-                runner_end (&runner);
+                gf_log ("glusterd", GF_LOG_DEBUG, "rebalance command failed");
                 goto out;
         }
 
         sleep (5);
-        ret = rpc_clnt_transport_unix_options_build (&options, sockfile);
+
+        ret = glusterd_rebalance_rpc_create (volinfo, _gf_false);
+
+        //FIXME: this cbk is passed as NULL in all occurrences. May be
+        //we never needed it.
+        if (cbk)
+                defrag->cbk_fn = cbk;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int
+glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
+                               gf_boolean_t reconnect)
+{
+        dict_t                  *options = NULL;
+        char                     sockfile[PATH_MAX] = {0,};
+        int                      ret = -1;
+        glusterd_defrag_info_t  *defrag = volinfo->rebal.defrag;
+        glusterd_conf_t         *priv = NULL;
+        xlator_t                *this = NULL;
+        struct stat             buf = {0,};
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        //rebalance process is not started
+        if (!defrag)
+                goto out;
+
+        //rpc obj for rebalance process already in place.
+        if (defrag->rpc) {
+                ret = 0;
+                goto out;
+        }
+        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
+        /* If reconnecting check if defrag sockfile exists in the new location
+         * in /var/run/ , if it does not try the old location
+         */
+        if (reconnect) {
+                ret = sys_stat (sockfile, &buf);
+                /* TODO: Remove this once we don't need backward compatability
+                 * with the older path
+                 */
+                if (ret && (errno == ENOENT)) {
+                        gf_log (this->name, GF_LOG_WARNING, "Rebalance sockfile "
+                                "%s does not exist. Trying old path.",
+                                sockfile);
+                        GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD (sockfile, volinfo,
+                                                           priv);
+                        ret =sys_stat (sockfile, &buf);
+                        if (ret && (ENOENT == errno)) {
+                                gf_log (this->name, GF_LOG_ERROR, "Rebalance "
+                                        "sockfile %s does not exist.",
+                                        sockfile);
+                                goto out;
+                        }
+                }
+        }
+
+        /* Setting frame-timeout to 10mins (600seconds).
+         * Unix domain sockets ensures that the connection is reliable. The
+         * default timeout of 30mins used for unreliable network connections is
+         * too long for unix domain socket connections.
+         */
+        ret = rpc_transport_unix_options_build (&options, sockfile, 600);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_ERROR, "Unix options build failed");
                 goto out;
         }
 
+        glusterd_volinfo_ref (volinfo);
+        synclock_unlock (&priv->big_lock);
         ret = glusterd_rpc_create (&defrag->rpc, options,
                                    glusterd_defrag_notify, volinfo);
+        synclock_lock (&priv->big_lock);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
                 goto out;
         }
-
-        if (cbk)
-                defrag->cbk_fn = cbk;
-
+        ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -396,26 +416,32 @@ out:
 }
 
 int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+__glusterd_handle_defrag_volume (rpcsvc_request_t *req)
 {
-        int32_t                 ret     = -1;
-        gf_cli_req              cli_req = {{0,}};
-        glusterd_conf_t        *priv    = NULL;
-        dict_t                 *dict    = NULL;
-        char                   *volname = NULL;
-        gf_cli_defrag_type      cmd     = 0;
+        int32_t                 ret       = -1;
+        gf_cli_req              cli_req   = {{0,}};
+        glusterd_conf_t        *priv      = NULL;
+        dict_t                 *dict      = NULL;
+        char                   *volname   = NULL;
+        gf_cli_defrag_type      cmd       = 0;
+        char                    msg[2048] = {0,};
+        xlator_t               *this      = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        priv = THIS->private;
+        priv = this->private;
+        GF_ASSERT (priv);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
-       if (cli_req.dict.dict_len) {
+
+        if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
                 dict  = dict_new ();
 
@@ -423,40 +449,39 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to get volname");
+                snprintf (msg, sizeof (msg), "Failed to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "rebalance-command", (int32_t*)&cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to get command");
+                snprintf (msg, sizeof (msg), "Failed to get command");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
-        glusterd_rebalance_cmd_attempted_log (cmd, volname);
-
-        ret = dict_set_static_bin (dict, "node-uuid", priv->uuid, 16);
+        ret = dict_set_static_bin (dict, "node-uuid", MY_UUID, 16);
         if (ret)
                 goto out;
 
         if ((cmd == GF_DEFRAG_CMD_STATUS) ||
               (cmd == GF_DEFRAG_CMD_STOP)) {
                 ret = glusterd_op_begin (req, GD_OP_DEFRAG_BRICK_VOLUME,
-                                         dict);
-        }
-        else
-                ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict);
+                                         dict, msg, sizeof (msg));
+        } else
+                ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict,
+                                         msg, sizeof (msg));
 
 out:
 
@@ -464,83 +489,100 @@ out:
         glusterd_op_sm ();
 
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
-                ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0, req,
-                                                     NULL, "operation failed");
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
+                ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0,
+                                                     req, dict, msg);
+
         }
 
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);//malloced by xdr
+        free (cli_req.dict.dict_val);//malloced by xdr
 
         return 0;
 }
 
+int
+glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_defrag_volume);
+}
+
 
 int
 glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
 {
-        char *volname = NULL;
-        int ret = 0;
-        int32_t cmd = 0;
-        char msg[2048] = {0};
-        glusterd_volinfo_t  *volinfo = NULL;
+        char                    *volname = NULL;
+        int                     ret = 0;
+        int32_t                 cmd = 0;
+        char                    msg[2048] = {0};
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *task_id_str = NULL;
+        dict_t                  *op_ctx = NULL;
+        xlator_t                *this = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "volname not found");
+                gf_log (this->name, GF_LOG_DEBUG, "volname not found");
                 goto out;
         }
+
         ret = dict_get_int32 (dict, "rebalance-command", &cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "cmd not found");
+                gf_log (this->name, GF_LOG_DEBUG, "cmd not found");
                 goto out;
         }
 
         ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
                                                msg, sizeof (msg));
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "failed to validate");
+                gf_log (this->name, GF_LOG_DEBUG, "failed to validate");
                 goto out;
         }
         switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_FORCE:
-                ret = glusterd_defrag_start_validate (volinfo,
-                                                      msg, sizeof (msg));
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "start validate failed");
-                        goto out;
-                }
-                break;
-        case GF_DEFRAG_CMD_STATUS:
-                ret = glusterd_is_defrag_on (volinfo);
-                if (!ret) {
-                        ret = -1;
-                        if (volinfo->defrag_status ==
-                                GF_DEFRAG_STATUS_COMPLETE) {
-                                snprintf (msg, sizeof (msg), "Rebalance "
-                                          "completed!");
+                if (is_origin_glusterd (dict)) {
+                        op_ctx = glusterd_op_get_ctx ();
+                        if (!op_ctx) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                goto out;
+                        }
+
+                        ret = glusterd_generate_and_set_task_id
+                                (op_ctx, GF_REBALANCE_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
                                 goto out;
                         }
-                        snprintf (msg, sizeof(msg), "Rebalance is not running"
-                                  " on volume %s", volname);
+                } else {
+                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                snprintf (msg, sizeof (msg),
+                                          "Missing rebalance-id");
+                                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
+                                ret = 0;
+                        }
+                }
+                ret = glusterd_defrag_start_validate (volinfo, msg,
+                                                      sizeof (msg),
+                                                      GD_OP_REBALANCE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                        "start validate failed");
                         goto out;
                 }
                 break;
-
+        case GF_DEFRAG_CMD_STATUS:
         case GF_DEFRAG_CMD_STOP:
-                ret = glusterd_is_defrag_on (volinfo);
-                if (!ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "rebalance is not running");
-                        ret = -1;
-                        snprintf (msg, sizeof(msg), "Rebalance is not running"
-                                  " on volume %s", volname);
-                        goto out;
-                }
+                break;
         default:
                 break;
         }
@@ -557,56 +599,172 @@ out:
 int
 glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char               *volname   = NULL;
-        int                 ret       = 0;
-        int32_t             cmd       = 0;
-        char                msg[2048] = {0};
-        glusterd_volinfo_t *volinfo   = NULL;
-        glusterd_conf_t    *priv      = NULL;
-
-        priv = THIS->private;
+        char                    *volname   = NULL;
+        int                     ret       = 0;
+        int32_t                 cmd       = 0;
+        char                    msg[2048] = {0};
+        glusterd_volinfo_t      *volinfo   = NULL;
+        glusterd_conf_t         *priv      = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_brickinfo_t    *tmp      = NULL;
+        gf_boolean_t            volfile_update = _gf_false;
+        char                    *task_id_str = NULL;
+        dict_t                  *ctx = NULL;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "volname not given");
+                gf_log (this->name, GF_LOG_DEBUG, "volname not given");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "rebalance-command", &cmd);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "command not given");
+                gf_log (this->name, GF_LOG_DEBUG, "command not given");
                 goto out;
         }
 
+
         ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
                                                msg, sizeof (msg));
         if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "cmd validate failed");
+                gf_log (this->name, GF_LOG_DEBUG, "cmd validate failed");
                 goto out;
         }
 
+        /* Set task-id, if available, in op_ctx dict for operations other than
+         * start
+         */
+        if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
+                if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                        ctx = glusterd_op_get_ctx ();
+                        if (!ctx) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                ret = -1;
+                                goto out;
+                        }
+
+                        if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rebal.rebalance_id, ctx,
+                                         GF_REMOVE_BRICK_TID_KEY);
+                        else
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rebal.rebalance_id, ctx,
+                                         GF_REBALANCE_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set task-id");
+                                goto out;
+                        }
+                }
+        }
+
         switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_FORCE:
+                /* Reset defrag status to 'NOT STARTED' whenever a
+                 * remove-brick/rebalance command is issued to remove
+                 * stale information from previous run.
+                 */
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
+
+                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Missing rebalance "
+                                "id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
+                        volinfo->rebal.op = GD_OP_REBALANCE;
+                }
+                if (!gd_should_i_start_rebalance (volinfo))
+                        break;
                 ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
-                                                    cmd, NULL);
-                 break;
+                                                    cmd, NULL, GD_OP_REBALANCE);
+                break;
         case GF_DEFRAG_CMD_STOP:
+                /* Clear task-id only on explicitly stopping rebalance.
+                 * Also clear the stored operation, so it doesn't cause trouble
+                 * with future rebalance/remove-brick starts
+                 */
+                uuid_clear (volinfo->rebal.rebalance_id);
+                volinfo->rebal.op = GD_OP_NONE;
+
+                /* Fall back to the old volume file in case of decommission*/
+                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
+                                          brick_list) {
+                        if (!brickinfo->decommissioned)
+                                continue;
+                        brickinfo->decommissioned = 0;
+                        volfile_update = _gf_true;
+                }
+
+                if (volfile_update == _gf_false) {
+                        ret = 0;
+                        break;
+                }
+
+                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to create volfiles");
+                        goto out;
+                }
+
+                ret = glusterd_store_volinfo (volinfo,
+                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to store volinfo");
+                        goto out;
+                }
+
+                ret = 0;
+                break;
+
         case GF_DEFRAG_CMD_STATUS:
-                volinfo->rebalance_files = 0;
-                volinfo->rebalance_data = 0;
-                volinfo->lookedup_files = 0;
                 break;
         default:
                 break;
         }
 
-        glusterd_rebalance_cmd_log (cmd, volname, ret);
-
 out:
         if (ret && op_errstr && msg[0])
                 *op_errstr = gf_strdup (msg);
 
         return ret;
 }
+
+int32_t
+glusterd_defrag_event_notify_handle (dict_t *dict)
+{
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *volname = NULL;
+        int32_t                  ret     = -1;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to get volname");
+                return ret;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to get volinfo for %s"
+                        , volname);
+                return ret;
+        }
+
+        ret = glusterd_defrag_volume_status_update (volinfo, dict);
+
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Failed to update status");
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 01bc974b7..e78eff44d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -36,6 +26,13 @@
 
 #include <signal.h>
 
+#define GLUSTERD_GET_RB_MNTPT(path, len, volinfo)                           \
+        snprintf (path, len,                                                \
+                  DEFAULT_VAR_RUN_DIRECTORY"/%s-"RB_CLIENT_MOUNTPOINT,      \
+                  volinfo->volname);
+
+extern uuid_t global_txn_id;
+
 int
 glusterd_get_replace_op_str (gf1_cli_replace_op op, char *op_str)
 {
@@ -74,7 +71,7 @@ out:
 }
 
 int
-glusterd_handle_replace_brick (rpcsvc_request_t *req)
+__glusterd_handle_replace_brick (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -85,17 +82,21 @@ glusterd_handle_replace_brick (rpcsvc_request_t *req)
         char                            operation[256];
         glusterd_op_t                   cli_op = GD_OP_REPLACE_BRICK;
         char                            *volname = NULL;
+        char                            msg[2048] = {0,};
+        xlator_t                        *this = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received replace brick req");
+        gf_log (this->name, GF_LOG_INFO, "Received replace brick req");
 
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
@@ -105,70 +106,79 @@ glusterd_handle_replace_brick (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "could not get volname");
+                snprintf (msg, sizeof (msg), "Could not get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "operation", &op);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "dict_get on operation failed");
+                snprintf (msg, sizeof (msg), "Could not get operation");
                 goto out;
         }
 
         ret = dict_get_str (dict, "src-brick", &src_brick);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get src brick");
+                snprintf (msg, sizeof (msg), "Failed to get src brick");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
-        gf_log ("", GF_LOG_DEBUG,
+        gf_log (this->name, GF_LOG_DEBUG,
                 "src brick=%s", src_brick);
 
         ret = dict_get_str (dict, "dst-brick", &dst_brick);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get dest brick");
+                snprintf (msg, sizeof (msg), "Failed to get dest brick");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 goto out;
         }
 
         (void) glusterd_get_replace_op_str (op, operation);
-        gf_log ("", GF_LOG_DEBUG, "dst brick=%s", dst_brick);
-        gf_log ("glusterd", GF_LOG_INFO, "Received replace brick %s request",
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
+        gf_log (this->name, GF_LOG_INFO, "Received replace brick %s request",
                 operation);
-        gf_cmd_log ("Volume replace-brick","volname: %s src_brick:%s"
-                    " dst_brick:%s op:%s", volname, src_brick, dst_brick,
-                    operation);
 
-        ret = glusterd_op_begin (req, GD_OP_REPLACE_BRICK, dict);
-        gf_cmd_log ("Volume replace-brick","on volname: %s %s", volname,
-                   (ret) ? "FAILED" : "SUCCESS");
+        ret = glusterd_op_begin (req, GD_OP_REPLACE_BRICK, dict,
+                                 msg, sizeof (msg));
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);//malloced by xdr
+        free (cli_req.dict.dict_val);//malloced by xdr
 
         glusterd_friend_sm ();
         glusterd_op_sm ();
 
-        if (ret)
+        if (ret) {
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, msg);
+        }
 
         return ret;
 }
 
+int
+glusterd_handle_replace_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_replace_brick);
+}
+
 static int
 glusterd_get_rb_dst_brickinfo (glusterd_volinfo_t *volinfo,
                                glusterd_brickinfo_t **brickinfo)
@@ -178,7 +188,7 @@ glusterd_get_rb_dst_brickinfo (glusterd_volinfo_t *volinfo,
         if (!volinfo || !brickinfo)
                 goto out;
 
-        *brickinfo = volinfo->dst_brick;
+        *brickinfo = volinfo->rep_brick.dst_brick;
 
         ret = 0;
 
@@ -191,6 +201,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                                  dict_t *rsp_dict)
 {
         int                                      ret           = 0;
+        int32_t                                  port          = 0;
         char                                    *src_brick     = NULL;
         char                                    *dst_brick     = NULL;
         char                                    *volname       = NULL;
@@ -199,47 +210,53 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         glusterd_brickinfo_t                    *src_brickinfo = NULL;
         char                                    *host          = NULL;
         char                                    *path          = NULL;
-        char                                    msg[2048]      = {0};
+        char                                     msg[2048]     = {0};
         char                                    *dup_dstbrick  = NULL;
         glusterd_peerinfo_t                     *peerinfo = NULL;
         glusterd_brickinfo_t                    *dst_brickinfo = NULL;
-        gf_boolean_t                            is_run         = _gf_false;
+        gf_boolean_t                             is_run        = _gf_false;
         dict_t                                  *ctx           = NULL;
         glusterd_conf_t                         *priv          = NULL;
-        char                                    voldir[PATH_MAX] = {0};
-        char                                    pidfile[PATH_MAX] = {0};
+        char                                    *savetok       = NULL;
+        char                                     pidfile[PATH_MAX] = {0};
+        char                                    *task_id_str = NULL;
+        xlator_t                                *this = NULL;
+        gf_boolean_t                            is_force = _gf_false;
 
-        priv = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
         GF_ASSERT (priv);
 
         ret = dict_get_str (dict, "src-brick", &src_brick);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get src brick");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get src brick");
                 goto out;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "src brick=%s", src_brick);
+        gf_log (this->name, GF_LOG_DEBUG, "src brick=%s", src_brick);
 
         ret = dict_get_str (dict, "dst-brick", &dst_brick);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get dest brick");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get dest brick");
                 goto out;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "dst brick=%s", dst_brick);
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
 
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "operation", (int32_t *)&replace_op);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "dict get on replace-brick operation failed");
                 goto out;
         }
@@ -264,7 +281,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 !glusterd_is_valid_volfpath (volname, dst_brick)) {
                 snprintf (msg, sizeof (msg), "brick path %s is too "
                           "long.", dst_brick);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 *op_errstr = gf_strdup (msg);
 
                 ret = -1;
@@ -273,10 +290,10 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
 
         ret = glusterd_check_gsync_running (volinfo, &is_run);
         if (ret && (is_run == _gf_false))
-                gf_log ("", GF_LOG_WARNING, "Unable to get the status"
+                gf_log (this->name, GF_LOG_WARNING, "Unable to get the status"
                                 " of active "GEOREP" session");
         if (is_run) {
-                gf_log ("", GF_LOG_WARNING, GEOREP" sessions active"
+                gf_log (this->name, GF_LOG_WARNING, GEOREP" sessions active"
                         "for the volume %s ", volname);
                 snprintf (msg, sizeof(msg), GEOREP" sessions are active "
                                 "for the volume %s.\nStop "GEOREP " sessions "
@@ -291,29 +308,61 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         if (glusterd_is_defrag_on(volinfo)) {
                 snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
                           "progress. Please retry after completion", volname);
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
         }
 
+        ctx = glusterd_op_get_ctx();
+
         switch (replace_op) {
         case GF_REPLACE_OP_START:
                 if (glusterd_is_rb_started (volinfo)) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is already "
-                                "started for volume ");
+                        snprintf (msg, sizeof (msg), "Replace brick is already "
+                                  "started for volume");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
                         ret = -1;
                         goto out;
                 }
+                if (is_origin_glusterd (dict)) {
+                        if (!ctx) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                goto out;
+                        }
+
+                        ret = glusterd_generate_and_set_task_id
+                                (ctx, GF_REPLACE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
+                                goto out;
+                        }
+
+                } else {
+                        ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Missing replace-brick-id");
+                                ret = 0;
+                        }
+                }
+                is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
                 break;
+
         case GF_REPLACE_OP_PAUSE:
                 if (glusterd_is_rb_paused (volinfo)) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is already "
-                                "paused for volume ");
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is "
+                                "already paused for volume ");
                         ret = -1;
                         goto out;
                 } else if (!glusterd_is_rb_started(volinfo)) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is not"
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not"
                                 " started for volume ");
                         ret = -1;
                         goto out;
@@ -321,9 +370,8 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 break;
 
         case GF_REPLACE_OP_ABORT:
-                if ((!glusterd_is_rb_paused (volinfo)) &&
-                     (!glusterd_is_rb_started (volinfo))) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is not"
+                if (!glusterd_is_rb_ongoing (volinfo)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not"
                                 " started or paused for volume ");
                         ret = -1;
                         goto out;
@@ -331,15 +379,17 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 break;
 
         case GF_REPLACE_OP_COMMIT:
-                if (!glusterd_is_rb_started (volinfo)) {
-                        gf_log ("", GF_LOG_ERROR, "Replace brick is not "
+                if (!glusterd_is_rb_ongoing (volinfo)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not "
                                 "started for volume ");
                         ret = -1;
                         goto out;
                 }
                 break;
 
-        case GF_REPLACE_OP_COMMIT_FORCE: break;
+        case GF_REPLACE_OP_COMMIT_FORCE:
+                is_force = _gf_true;
+                break;
 
         case GF_REPLACE_OP_STATUS:
 
@@ -352,7 +402,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                                 goto out;
                         }
 
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
                         ret = -1;
                         goto out;
                 }
@@ -364,8 +414,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         }
 
         ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo,
-                                                      GF_PATH_COMPLETE);
+                                                      &src_brickinfo);
         if (ret) {
                 snprintf (msg, sizeof (msg), "brick: %s does not exist in "
                           "volume: %s", src_brick, volname);
@@ -373,10 +422,9 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
-        ctx = glusterd_op_get_ctx();
         if (ctx) {
                 if (!glusterd_is_fuse_available ()) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to open /dev/"
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to open /dev/"
                                 "fuse (%s), replace-brick command failed",
                                 strerror (errno));
                         snprintf (msg, sizeof(msg), "Fuse unavailable\n "
@@ -387,8 +435,8 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 }
         }
 
-        if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
-                gf_log ("", GF_LOG_DEBUG,
+        if (gf_is_local_addr (src_brickinfo->hostname)) {
+                gf_log (this->name, GF_LOG_DEBUG,
                         "I AM THE SOURCE HOST");
                 if (src_brickinfo->port && rsp_dict) {
                         ret = dict_set_int32 (rsp_dict, "src-brick-port",
@@ -400,11 +448,10 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         }
                 }
 
-                GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv);
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, voldir,
-                                            src_brickinfo->hostname,
-                                            src_brickinfo->path);
-                if (!glusterd_is_service_running (pidfile, NULL)) {
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, src_brickinfo,
+                                            priv);
+                if ((replace_op != GF_REPLACE_OP_COMMIT_FORCE) &&
+                    !gf_is_service_running (pidfile, NULL)) {
                         snprintf(msg, sizeof(msg), "Source brick %s:%s "
                                  "is not online.", src_brickinfo->hostname,
                                  src_brickinfo->path);
@@ -419,53 +466,61 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         dup_dstbrick = gf_strdup (dst_brick);
         if (!dup_dstbrick) {
                 ret = -1;
-                gf_log ("", GF_LOG_ERROR, "Memory allocation failed");
+                gf_log (this->name, GF_LOG_ERROR, "Memory allocation failed");
                 goto out;
         }
-        host = strtok (dup_dstbrick, ":");
-        path = strtok (NULL, ":");
+        host = strtok_r (dup_dstbrick, ":", &savetok);
+        path = strtok_r (NULL, ":", &savetok);
 
         if (!host || !path) {
-                gf_log ("", GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "dst brick %s is not of form <HOSTNAME>:<export-dir>",
                         dst_brick);
                 ret = -1;
                 goto out;
         }
-        if (!glusterd_brickinfo_get (NULL, host, path, NULL)) {
-                snprintf(msg, sizeof(msg), "Brick: %s:%s already in use",
-                         host, path);
+
+        ret = glusterd_brickinfo_new_from_brick (dst_brick, &dst_brickinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_new_brick_validate (dst_brick, dst_brickinfo,
+                                           msg, sizeof (msg));
+        if (ret) {
                 *op_errstr = gf_strdup (msg);
                 ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
                 goto out;
         }
 
-        if ((volinfo->rb_status ==GF_RB_STATUS_NONE) &&
-            (replace_op == GF_REPLACE_OP_START)) {
-                ret = glusterd_brickinfo_from_brick (dst_brick, &dst_brickinfo);
-                volinfo->src_brick = src_brickinfo;
-                volinfo->dst_brick = dst_brickinfo;
-        } else {
-                ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
+       if (!glusterd_is_rb_ongoing (volinfo) &&
+            (replace_op == GF_REPLACE_OP_START ||
+             replace_op == GF_REPLACE_OP_COMMIT_FORCE)) {
+
+                volinfo->rep_brick.src_brick = src_brickinfo;
+                volinfo->rep_brick.dst_brick = dst_brickinfo;
         }
 
         if (glusterd_rb_check_bricks (volinfo, src_brickinfo, dst_brickinfo)) {
 
                 ret = -1;
-                *op_errstr = gf_strdup ("incorrect source or "
+                *op_errstr = gf_strdup ("Incorrect source or "
                                         "destination brick");
                 if (*op_errstr)
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
                 goto out;
        }
 
-        if (!glusterd_is_local_addr (host)) {
-                ret = glusterd_brick_create_path (host, path,
-                                                  volinfo->volume_id, 0777,
-                                                  op_errstr);
+        if (!glusterd_is_rb_ongoing (volinfo) &&
+            gf_is_local_addr (host)) {
+                ret = glusterd_validate_and_create_brickpath (dst_brickinfo,
+                                                  volinfo->volume_id,
+                                                  op_errstr, is_force);
                 if (ret)
                         goto out;
-        } else {
+        }
+
+        if (!gf_is_local_addr (host)) {
                 ret = glusterd_friend_find (NULL, host, &peerinfo);
                 if (ret) {
                         snprintf (msg, sizeof (msg), "%s, is not a friend",
@@ -490,13 +545,72 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         goto out;
                 }
         }
+
+        if (replace_op == GF_REPLACE_OP_START &&
+            gf_is_local_addr (volinfo->rep_brick.dst_brick->hostname)) {
+                port = pmap_registry_alloc (THIS);
+                if (!port) {
+                        gf_log (THIS->name, GF_LOG_CRITICAL,
+                                "No free ports available");
+                        ret = -1;
+                        goto out;
+                }
+
+                ctx = glusterd_op_get_ctx();
+                ret = dict_set_int32 ((ctx)?ctx:rsp_dict, "dst-brick-port",
+                                      port);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to set dst "
+                                "brick port");
+                        goto out;
+                }
+                volinfo->rep_brick.dst_brick->port = port;
+        }
+
         ret = 0;
 
 out:
-        if (dup_dstbrick)
-                GF_FREE (dup_dstbrick);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        GF_FREE (dup_dstbrick);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+rb_set_mntfd (int mntfd)
+{
+        int     ret = -1;
+        dict_t *ctx = NULL;
+
+        ctx = glusterd_op_get_ctx ();
+        if (!ctx) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Failed to get op ctx");
+                goto out;
+        }
+        ret = dict_set_int32 (ctx, "mntfd", mntfd);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_DEBUG, "Failed to set mnt fd "
+                        "in op ctx");
+out:
+        return ret;
+}
 
+static int
+rb_get_mntfd (int *mntfd)
+{
+        int     ret = -1;
+        dict_t *ctx = NULL;
+
+        ctx = glusterd_op_get_ctx ();
+        if (!ctx) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Failed to get op ctx");
+                goto out;
+        }
+        ret = dict_get_int32 (ctx, "mntfd", mntfd);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_DEBUG, "Failed to get mnt fd "
+                        "from op ctx");
+out:
         return ret;
 }
 
@@ -522,6 +636,8 @@ rb_regenerate_volfiles (glusterd_volinfo_t *volinfo,
 
         ret = glusterd_create_rb_volfiles (volinfo, brickinfo);
 
+        dict_del (dict, "enable-pump");
+
 out:
         return ret;
 }
@@ -543,7 +659,8 @@ rb_src_brick_restart (glusterd_volinfo_t *volinfo,
                         ret);
         }
 
-        ret = glusterd_volume_stop_glusterfs (volinfo, src_brickinfo);
+        ret = glusterd_volume_stop_glusterfs (volinfo, src_brickinfo,
+                                              _gf_false);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to stop "
                         "glusterfs, ret: %d", ret);
@@ -570,7 +687,8 @@ rb_src_brick_restart (glusterd_volinfo_t *volinfo,
         }
 
         sleep (2);
-        ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo);
+        ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo,
+                                              _gf_false);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to start "
                         "glusterfs, ret: %d", ret);
@@ -592,22 +710,19 @@ rb_send_xattr_command (glusterd_volinfo_t *volinfo,
                        glusterd_brickinfo_t *dst_brickinfo,
                        const char *xattr_key, const char *value)
 {
-        glusterd_conf_t  *priv                        = NULL;
-        xlator_t         *this                        = NULL;
-        char              mount_point_path[PATH_MAX]  = {0,};
-        int               ret                         = -1;
-
-        this = THIS;
-        priv = this->private;
+        int             ret   = -1;
+        int             mntfd = -1;
 
-        snprintf (mount_point_path, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname, RB_CLIENT_MOUNTPOINT);
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
 
-        ret = sys_lsetxattr (mount_point_path, xattr_key, value,
-                             strlen (value) + 1, 0);
+        ret = sys_fsetxattr (mntfd, xattr_key, value, strlen (value) + 1, 0);
         if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "setxattr on key: "
-                        "%s failed", xattr_key);
+                gf_log (THIS->name, GF_LOG_DEBUG, "setxattr on key: "
+                        "%s, reason: %s", xattr_key, strerror (errno));
+
+out:
         return ret;
 }
 
@@ -622,9 +737,7 @@ rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
 
         priv = THIS->private;
 
-        port = pmap_registry_alloc (THIS);
-        brickinfo->port = port;
-
+        port = brickinfo->port;
         GF_ASSERT (port);
 
         runinit (&runner);
@@ -635,9 +748,13 @@ rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
                           priv->workdir, volinfo->volname);
         runner_add_arg (&runner, "--xlator-option");
         runner_argprintf (&runner, "src-server.listen-port=%d", port);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
 
-        ret = runner_run (&runner);
+        ret = runner_run_nowait (&runner);
         if (ret) {
+                pmap_registry_remove (THIS, 0, brickinfo->path,
+                                      GF_PMAP_PORT_BRICKSERVER, NULL);
                 gf_log ("", GF_LOG_DEBUG,
                         "Could not start glusterfs");
                 goto out;
@@ -657,51 +774,70 @@ static int
 rb_spawn_glusterfs_client (glusterd_volinfo_t *volinfo,
                            glusterd_brickinfo_t *brickinfo)
 {
-        glusterd_conf_t    *priv           = NULL;
-        char                cmd_str[8192]  = {0,};
-        runner_t            runner         = {0,};
-        struct stat         buf;
-        int                 ret            = -1;
+        xlator_t           *this            = NULL;
+        glusterd_conf_t    *priv            = NULL;
+        runner_t            runner          = {0,};
+        struct stat         buf             = {0,};
+        char                mntpt[PATH_MAX] = {0,};
+        int                 mntfd           = -1;
+        int                 ret             = -1;
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
 
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
         runinit (&runner);
         runner_add_arg (&runner, SBIN_DIR"/glusterfs");
         runner_argprintf (&runner, "-f" "%s/vols/%s/"RB_CLIENTVOL_FILENAME,
                           priv->workdir, volinfo->volname);
-        runner_argprintf (&runner, "%s/vols/%s/"RB_CLIENT_MOUNTPOINT,
-                          priv->workdir, volinfo->volname);
+        runner_add_arg (&runner, mntpt);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
 
-        ret = runner_run (&runner);
+        ret = runner_run_reuse (&runner);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Could not start glusterfs");
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Could not start glusterfs");
+                runner_end (&runner);
                 goto out;
+        } else {
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Successfully started  glusterfs");
+                runner_end (&runner);
         }
 
-        gf_log ("", GF_LOG_DEBUG,
-                "Successfully started glusterfs: brick=%s:%s",
-                brickinfo->hostname, brickinfo->path);
+        ret = stat (mntpt, &buf);
+        if (ret) {
+                 gf_log (this->name, GF_LOG_DEBUG, "stat on mount point %s "
+                         "failed", mntpt);
+                 goto out;
+        }
 
-        memset (cmd_str, 0, sizeof (cmd_str));
+        mntfd = open (mntpt, O_DIRECTORY);
+        if (mntfd == -1)
+                goto out;
 
-        snprintf (cmd_str, 4096, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENT_MOUNTPOINT);
+        ret = rb_set_mntfd (mntfd);
+        if (ret)
+                goto out;
 
-        ret = stat (cmd_str, &buf);
+        runinit (&runner);
+        runner_add_args (&runner, "/bin/umount", "-l", mntpt, NULL);
+        ret = runner_run_reuse (&runner);
         if (ret) {
-                 gf_log ("", GF_LOG_DEBUG,
-                         "stat on mountpoint failed");
-                 goto out;
-         }
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Lazy unmount failed on maintenance client");
+                runner_end (&runner);
+                goto out;
+        } else {
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Successfully unmounted  maintenance client");
+                runner_end (&runner);
+        }
 
-         gf_log ("", GF_LOG_DEBUG,
-                 "stat on mountpoint succeeded");
-
-        ret = 0;
 
 out:
+
         return ret;
 }
 
@@ -868,20 +1004,14 @@ static int
 rb_mountpoint_mkdir (glusterd_volinfo_t *volinfo,
                      glusterd_brickinfo_t *src_brickinfo)
 {
-        glusterd_conf_t *priv                       = NULL;
-        char             mount_point_path[PATH_MAX] = {0,};
-        int              ret                        = -1;
+        char             mntpt[PATH_MAX]        = {0,};
+        int              ret                    = -1;
 
-        priv = THIS->private;
-
-        snprintf (mount_point_path, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENT_MOUNTPOINT);
-
-        ret = mkdir (mount_point_path, 0777);
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
+        ret = mkdir (mntpt, 0777);
         if (ret && (errno != EEXIST)) {
-                gf_log ("", GF_LOG_DEBUG, "mkdir failed, errno: %d",
-                        errno);
+                gf_log ("", GF_LOG_DEBUG, "mkdir failed, due to %s",
+                        strerror (errno));
                 goto out;
         }
 
@@ -895,19 +1025,14 @@ static int
 rb_mountpoint_rmdir (glusterd_volinfo_t *volinfo,
                      glusterd_brickinfo_t *src_brickinfo)
 {
-        glusterd_conf_t *priv                       = NULL;
-        char             mount_point_path[PATH_MAX] = {0,};
+        char             mntpt[PATH_MAX] = {0,};
         int              ret                        = -1;
 
-        priv = THIS->private;
-
-        snprintf (mount_point_path, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENT_MOUNTPOINT);
-
-        ret = rmdir (mount_point_path);
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
+        ret = rmdir (mntpt);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "rmdir failed");
+                gf_log ("", GF_LOG_DEBUG, "rmdir failed, reason: %s",
+                        strerror (errno));
                 goto out;
         }
 
@@ -919,42 +1044,42 @@ out:
 
 static int
 rb_destroy_maintenance_client (glusterd_volinfo_t *volinfo,
-                                glusterd_brickinfo_t *src_brickinfo)
+                               glusterd_brickinfo_t *src_brickinfo)
 {
+        xlator_t         *this                        = NULL;
         glusterd_conf_t  *priv                        = NULL;
-        runner_t          runner                      = {0,};
-        char              filename[PATH_MAX]          = {0,};
+        char              volfile[PATH_MAX]           = {0,};
         int               ret                         = -1;
+        int               mntfd                       = -1;
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
 
-        runinit (&runner);
-        runner_add_args (&runner, "/bin/umount", "-f", NULL);
-        runner_argprintf (&runner, "%s/vols/%s/"RB_CLIENT_MOUNTPOINT,
-                          priv->workdir, volinfo->volname);
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
 
-        ret = runner_run (&runner);
+        ret = close (mntfd);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "umount failed on maintenance client");
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close mount "
+                        "point directory");
                 goto out;
         }
 
         ret = rb_mountpoint_rmdir (volinfo, src_brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "rmdir of mountpoint failed");
+                gf_log (this->name, GF_LOG_DEBUG, "rmdir of mountpoint "
+                        "failed");
                 goto out;
         }
 
-        snprintf (filename, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENTVOL_FILENAME);
+        snprintf (volfile, PATH_MAX, "%s/vols/%s/%s", priv->workdir,
+                  volinfo->volname, RB_CLIENTVOL_FILENAME);
 
-        ret = unlink (filename);
+        ret = unlink (volfile);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "unlink failed");
+                gf_log ("", GF_LOG_DEBUG, "unlink of %s failed, reason: %s",
+                        volfile, strerror (errno));
                 goto out;
         }
 
@@ -1043,23 +1168,18 @@ rb_get_xattr_command (glusterd_volinfo_t *volinfo,
                       const char *xattr_key,
                       char *value)
 {
-        glusterd_conf_t  *priv                        = NULL;
-        xlator_t         *this                        = NULL;
-        char              mount_point_path[PATH_MAX]  = {0,};
-        int               ret                         = -1;
-
-        this = THIS;
-        priv = THIS->private;
+        int             ret    = -1;
+        int             mntfd  = -1;
 
-        snprintf (mount_point_path, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENT_MOUNTPOINT);
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
 
-        ret = sys_lgetxattr (mount_point_path, xattr_key, value, 8192);
+        ret = sys_fgetxattr (mntfd, xattr_key, value, 8192);
 
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "getxattr on key: %s "
-                        "failed", xattr_key);
+                gf_log (THIS->name, GF_LOG_DEBUG, "getxattr on key: %s "
+                        "failed, reason: %s", xattr_key, strerror (errno));
                 goto out;
         }
 
@@ -1074,11 +1194,17 @@ rb_send_cmd (glusterd_volinfo_t *volinfo,
              glusterd_brickinfo_t *dst,
              gf1_cli_replace_op op)
 {
-        char         start_value[8192] = {0,};
-        char         status[8192]      = {0,};
-        char        *status_reply      = NULL;
-        dict_t      *ctx               = NULL;
-        int          ret               = 0;
+        char         start_value[8192]          = {0,};
+        char         status_str[8192]           = {0,};
+        char        *status_reply               = NULL;
+        char        *tmp                        = NULL;
+        char        *save_ptr                   = NULL;
+        char         filename[PATH_MAX]         = {0,};
+        char        *current_file               = NULL;
+        uint64_t     files                      = 0;
+        int          status                     = 0;
+        dict_t      *ctx                        = NULL;
+        int          ret                        = 0;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (src);
@@ -1122,7 +1248,7 @@ rb_send_cmd (glusterd_volinfo_t *volinfo,
                 {
                         ret = rb_get_xattr_command (volinfo, src, dst,
                                                     RB_PUMP_CMD_STATUS,
-                                                    status);
+                                                    status_str);
                         if (ret)
                                 goto out;
 
@@ -1135,7 +1261,76 @@ rb_send_cmd (glusterd_volinfo_t *volinfo,
                                 goto out;
                         }
 
-                        status_reply = gf_strdup (status);
+                        /* Split status reply into different parts */
+                        tmp = strtok_r (status_str, ":", &save_ptr);
+                        if (!tmp) {
+                                ret = -1;
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Couldn't tokenize status string");
+                                goto out;
+                        }
+                        sscanf (tmp, "status=%d", &status);
+                        ret = dict_set_int32 (ctx, "status", status);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
+                                        "set rb status in context");
+                                goto out;
+                        }
+
+                        tmp = NULL;
+                        tmp = strtok_r (NULL, ":", &save_ptr);
+                        if (!tmp) {
+                                ret = -1;
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Couldn't tokenize status string");
+                                goto out;
+                        }
+                        sscanf (tmp, "no_of_files=%"SCNu64, &files);
+                        ret = dict_set_uint64 (ctx, "files", files);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
+                                        "set rb files in context");
+                                goto out;
+                        }
+
+                        if (status == 0) {
+                                tmp = NULL;
+                                tmp = strtok_r (NULL, ":", &save_ptr);
+                                if (!tmp) {
+                                        ret = -1;
+                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                                "Couldn't tokenize status "
+                                                "string");
+                                        goto out;
+                                }
+                                sscanf (tmp, "current_file=%s", filename);
+                                current_file = gf_strdup (filename);
+                                ret = dict_set_dynstr (ctx, "current_file",
+                                                       current_file);
+                                if (ret) {
+                                        GF_FREE (current_file);
+                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                                "Couldn't set rb current file "
+                                                "in context");
+                                        goto out;
+                                }
+                        }
+                        if (status) {
+                                ret = gf_asprintf (&status_reply,
+                                                  "Number of files migrated = %"
+                                                  PRIu64"\tMigration complete",
+                                                  files);
+                        } else {
+                                ret = gf_asprintf (&status_reply,
+                                                  "Number of files migrated = %"
+                                                  PRIu64"\tCurrent file = %s",
+                                                  files, filename);
+                        }
+                        if (ret == -1) {
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Failed to create status_reply string");
+                                goto out;
+                        }
                         ret = dict_set_dynstr (ctx, "status-reply",
                                                status_reply);
                         if (ret) {
@@ -1209,14 +1404,11 @@ rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
 
         this = THIS;
 
-        ctx = glusterd_op_get_ctx ();
-        if (ctx) {
-                dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port);
-                if (src_port)
-                        src_brickinfo->port = src_port;
-        }
+        dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port);
+        if (src_port)
+                src_brickinfo->port = src_port;
 
-        if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
+        if (gf_is_local_addr (src_brickinfo->hostname)) {
                 gf_log ("", GF_LOG_INFO,
                         "adding src-brick port no");
 
@@ -1265,15 +1457,12 @@ rb_update_dstbrick_port (glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict,
         int     dict_ret      = 0;
         int     dst_port      = 0;
 
-        ctx = glusterd_op_get_ctx ();
-        if (ctx) {
-                dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
-                if (dst_port)
-                        dst_brickinfo->port = dst_port;
+        dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
+        if (!dict_ret)
+                dst_brickinfo->port = dst_port;
 
-        }
 
-        if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
+        if (gf_is_local_addr (dst_brickinfo->hostname)) {
                 gf_log ("", GF_LOG_INFO,
                         "adding dst-brick port no");
 
@@ -1312,21 +1501,24 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
 
         GF_ASSERT (volinfo);
 
-        ret = glusterd_brickinfo_from_brick (new_brick,
-                                             &new_brickinfo);
+        ret = glusterd_brickinfo_new_from_brick (new_brick,
+                                                 &new_brickinfo);
         if (ret)
                 goto out;
 
-        ret = glusterd_volume_brickinfo_get_by_brick (old_brick, volinfo,
-                                                      &old_brickinfo,
-                                                      GF_PATH_COMPLETE);
+        ret = glusterd_resolve_brick (new_brickinfo);
+
         if (ret)
                 goto out;
 
-        ret = glusterd_resolve_brick (new_brickinfo);
+        ret = glusterd_volume_brickinfo_get_by_brick (old_brick,
+                                                      volinfo, &old_brickinfo);
         if (ret)
                 goto out;
 
+        strncpy (new_brickinfo->brick_id, old_brickinfo->brick_id,
+                 sizeof (new_brickinfo->brick_id));
+
         list_add_tail (&new_brickinfo->brick_list,
                        &old_brickinfo->brick_list);
 
@@ -1341,13 +1533,13 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
                 goto out;
 
         if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                ret = glusterd_brick_start (volinfo, new_brickinfo);
+                ret = glusterd_brick_start (volinfo, new_brickinfo, _gf_false);
                 if (ret)
                         goto out;
         }
 
-
 out:
+
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -1366,6 +1558,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
         char                                    *dst_brick = NULL;
         glusterd_brickinfo_t                    *src_brickinfo = NULL;
         glusterd_brickinfo_t                    *dst_brickinfo = NULL;
+        char                                    *task_id_str = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1375,26 +1568,23 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
 
         ret = dict_get_str (dict, "src-brick", &src_brick);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get src brick");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get src brick");
                 goto out;
         }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "src brick=%s", src_brick);
+        gf_log (this->name, GF_LOG_DEBUG, "src brick=%s", src_brick);
 
         ret = dict_get_str (dict, "dst-brick", &dst_brick);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get dst brick");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get dst brick");
                 goto out;
         }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "dst brick=%s", dst_brick);
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
 
         ret = dict_get_str (dict, "volname", &volname);
-
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
@@ -1407,29 +1597,30 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
                 goto out;
         }
 
         ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo,
-                                                      GF_PATH_COMPLETE);
+                                                      &src_brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to get src-brickinfo");
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get src-brickinfo");
                 goto out;
         }
 
 
         ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get "
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get "
                          "replace brick destination brickinfo");
                 goto out;
         }
 
         ret = glusterd_resolve_brick (dst_brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to resolve dst-brickinfo");
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to resolve dst-brickinfo");
                 goto out;
         }
 
@@ -1438,45 +1629,82 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
         if (ret)
                 goto out;
 
+
 	if ((GF_REPLACE_OP_START != replace_op)) {
-                ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
-                                               dict, replace_op);
-                if (ret)
-                        goto out;
+
+                /* Set task-id, if available, in op_ctx dict for operations
+                 * other than start
+                 */
+                if  (is_origin_glusterd (dict)) {
+                        ctx = glusterd_op_get_ctx();
+                        if (!ctx) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get op_ctx");
+                                ret = -1;
+                                goto out;
+                        }
+                        if (!uuid_is_null (volinfo->rep_brick.rb_id)) {
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rep_brick.rb_id, ctx,
+                                         GF_REPLACE_BRICK_TID_KEY);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set "
+                                                "replace-brick-id");
+                                        goto out;
+                                }
+                        }
+                }
 	}
+        ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
+                                       dict, replace_op);
+        if (ret)
+                goto out;
 
         switch (replace_op) {
         case GF_REPLACE_OP_START:
         {
-                if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
-                        gf_log ("", GF_LOG_INFO,
+                ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Missing replace-brick-id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rep_brick.rb_id);
+                }
+
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_INFO,
                                 "I AM THE DESTINATION HOST");
                         if (!glusterd_is_rb_paused (volinfo)) {
-                                ret = rb_spawn_destination_brick (volinfo, dst_brickinfo);
+                                ret = rb_spawn_destination_brick
+                                        (volinfo, dst_brickinfo);
                                 if (ret) {
-                                        gf_log ("", GF_LOG_DEBUG,
-                                                "Failed to spawn destination brick");
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "Failed to spawn destination "
+                                                "brick");
                                         goto out;
                                 }
                         } else {
-                                gf_log ("", GF_LOG_ERROR, "Replace brick is already "
-                                        "started=> no need to restart dst brick ");
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Replace brick is already started=> no "
+                                        "need to restart dst brick ");
                         }
 		}
 
 
-		if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
+		if (gf_is_local_addr (src_brickinfo->hostname)) {
 		        ret = rb_src_brick_restart (volinfo, src_brickinfo,
 				                    1);
 		        if (ret) {
-				gf_log ("", GF_LOG_DEBUG,
-	                        "Could not restart src-brick");
+				gf_log (this->name, GF_LOG_DEBUG,
+	                                "Could not restart src-brick");
 			        goto out;
 			}
 		}
 
-		if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
-			gf_log ("", GF_LOG_INFO,
+		if (gf_is_local_addr (dst_brickinfo->hostname)) {
+			gf_log (this->name, GF_LOG_INFO,
 				"adding dst-brick port no");
 
                         ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
@@ -1490,7 +1718,6 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
 	}
 
         case GF_REPLACE_OP_COMMIT:
-        case GF_REPLACE_OP_COMMIT_FORCE:
         {
                 ctx = glusterd_op_get_ctx ();
                 if (ctx) {
@@ -1498,71 +1725,64 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                                                dst_brickinfo,
                                                GF_REPLACE_OP_COMMIT);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "Commit operation failed");
                                 goto out;
                         }
                 }
-
-                ret = dict_set_int32 (volinfo->dict, "enable-pump", 0);
-                gf_log ("", GF_LOG_DEBUG,
-                        "Received commit - will be adding dst brick and "
-                        "removing src brick");
-
-                if (!glusterd_is_local_addr (dst_brickinfo->hostname) &&
-                    replace_op != GF_REPLACE_OP_COMMIT_FORCE) {
-                        gf_log ("", GF_LOG_INFO,
+        }
+                /* fall through */
+        case GF_REPLACE_OP_COMMIT_FORCE:
+        {
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "I AM THE DESTINATION HOST");
-                        ret = rb_kill_destination_brick (volinfo, dst_brickinfo);
+                        ret = rb_kill_destination_brick (volinfo,
+                                                         dst_brickinfo);
                         if (ret) {
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Failed to kill destination brick");
+                                gf_log (this->name, GF_LOG_CRITICAL,
+                                        "Unable to cleanup dst brick");
                                 goto out;
                         }
                 }
 
-                if (ret) {
-                        gf_log ("", GF_LOG_CRITICAL,
-                                "Unable to cleanup dst brick");
-                        goto out;
-                }
-
-
                 ret = glusterd_nodesvcs_stop (volinfo);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Unable to stop nfs server, ret: %d", ret);
                 }
 
 		ret = glusterd_op_perform_replace_brick (volinfo, src_brick,
 							 dst_brick);
 		if (ret) {
-			gf_log ("", GF_LOG_CRITICAL, "Unable to add "
-				"dst-brick: %s to volume: %s",
-				dst_brick, volinfo->volname);
+			gf_log (this->name, GF_LOG_CRITICAL, "Unable to add "
+				"dst-brick: %s to volume: %s", dst_brick,
+                                volinfo->volname);
 		        (void) glusterd_nodesvcs_handle_graph_change (volinfo);
 			goto out;
 		}
 
-		volinfo->defrag_status = 0;
+		volinfo->rebal.defrag_status = 0;
 
 		ret = glusterd_nodesvcs_handle_graph_change (volinfo);
 		if (ret) {
-                        gf_log ("", GF_LOG_CRITICAL,
+                        gf_log (this->name, GF_LOG_CRITICAL,
                                 "Failed to generate nfs volume file");
 		}
 
 
 		ret = glusterd_fetchspec_notify (THIS);
                 glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE);
-                glusterd_brickinfo_delete (volinfo->dst_brick);
-                volinfo->src_brick = volinfo->dst_brick = NULL;
+                glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
+                volinfo->rep_brick.src_brick = NULL;
+                volinfo->rep_brick.dst_brick = NULL;
+                uuid_clear (volinfo->rep_brick.rb_id);
         }
         break;
 
         case GF_REPLACE_OP_PAUSE:
         {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Received pause - doing nothing");
                 ctx = glusterd_op_get_ctx ();
                 if (ctx) {
@@ -1570,7 +1790,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                                                dst_brickinfo,
                                                GF_REPLACE_OP_PAUSE);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "Pause operation failed");
                                 goto out;
                         }
@@ -1589,18 +1809,13 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                                                dst_brickinfo,
                                                GF_REPLACE_OP_ABORT);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "Abort operation failed");
                                 goto out;
                         }
                 }
 
-                ret = dict_set_int32 (volinfo->dict, "enable-pump", 0);
-		if (ret) {
-			gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to disable pump");
-		}
-
-                if (!glusterd_is_local_addr (src_brickinfo->hostname)) {
+                if (gf_is_local_addr (src_brickinfo->hostname)) {
                         ret = rb_src_brick_restart (volinfo, src_brickinfo,
                                                     0);
                         if (ret) {
@@ -1611,8 +1826,8 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                         }
                 }
 
-                if (!glusterd_is_local_addr (dst_brickinfo->hostname)) {
-                        gf_log (THIS->name, GF_LOG_INFO,
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_INFO,
                                 "I AM THE DESTINATION HOST");
                         ret = rb_kill_destination_brick (volinfo, dst_brickinfo);
                         if (ret) {
@@ -1622,14 +1837,15 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                         }
                 }
                 glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE);
-                glusterd_brickinfo_delete (volinfo->dst_brick);
-                volinfo->src_brick = volinfo->dst_brick = NULL;
+                glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
+                volinfo->rep_brick.src_brick = NULL;
+                volinfo->rep_brick.dst_brick = NULL;
         }
         break;
 
         case GF_REPLACE_OP_STATUS:
         {
-                gf_log ("", GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "received status - doing nothing");
                 ctx = glusterd_op_get_ctx ();
                 if (ctx) {
@@ -1637,7 +1853,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                                 ret = dict_set_str (ctx, "status-reply",
                                                  "replace brick has been paused");
                                 if (ret)
-                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                        gf_log (this->name, GF_LOG_ERROR,
                                                 "failed to set pump status"
                                                 " in ctx");
                                 goto out;
@@ -1661,7 +1877,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
 		ret = glusterd_store_volinfo (volinfo,
                                               GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't store"
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't store"
                         " replace brick operation's state");
 
 out:
@@ -1682,14 +1898,17 @@ glusterd_do_replace_brick (void *data)
         glusterd_brickinfo_t   *src_brickinfo = NULL;
         glusterd_brickinfo_t   *dst_brickinfo = NULL;
 	glusterd_conf_t	       *priv = NULL;
+        uuid_t                 *txn_id = NULL;
 
         int ret = 0;
 
         dict = data;
 
 	GF_ASSERT (THIS);
-
 	priv = THIS->private;
+        GF_ASSERT (priv);
+
+        txn_id = &priv->global_txn_id;
 
 	if (priv->timer) {
 		gf_timer_call_cancel (THIS->ctx, priv->timer);
@@ -1701,6 +1920,10 @@ glusterd_do_replace_brick (void *data)
         gf_log ("", GF_LOG_DEBUG,
                 "Replace brick operation detected");
 
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ret = dict_get_int32 (dict, "operation", &op);
         if (ret) {
                 gf_log ("", GF_LOG_DEBUG,
@@ -1739,8 +1962,7 @@ glusterd_do_replace_brick (void *data)
         }
 
         ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo,
-                                                      GF_PATH_COMPLETE);
+                                                      &src_brickinfo);
         if (ret) {
                 gf_log ("", GF_LOG_DEBUG, "Unable to get src-brickinfo");
                 goto out;
@@ -1797,12 +2019,15 @@ glusterd_do_replace_brick (void *data)
 
 out:
         if (ret)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                                   txn_id, NULL);
         else
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC, NULL);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC,
+                                                   txn_id, NULL);
 
-        if (dict)
-                dict_unref (dict);
-
-        glusterd_op_sm ();
+        synclock_lock (&priv->big_lock);
+        {
+                glusterd_op_sm ();
+        }
+        synclock_unlock (&priv->big_lock);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 4e55c383c..18f37c190 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -43,6 +33,7 @@
 
 
 extern glusterd_op_info_t opinfo;
+extern uuid_t global_txn_id;
 
 int32_t
 glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
@@ -59,10 +50,11 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         int32_t         status = 0;
         int32_t         count = 0;
         gf_cli_rsp      rsp = {0,};
+        xlator_t        *this = NULL;
 
-        GF_ASSERT (THIS);
-
-        conf = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
 
         GF_ASSERT (conf);
 
@@ -87,12 +79,13 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
                 if (ctx) {
                         ret = dict_get_int32 (ctx, "status", &status);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_TRACE,
+                                gf_log (this->name, GF_LOG_TRACE,
                                         "failed to get status");
                         }
                 }
                 break;
         }
+        case GD_OP_GSYNC_CREATE:
         case GD_OP_GSYNC_SET:
         {
                if (ctx) {
@@ -104,24 +97,12 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
                break;
 
         }
-        case GD_OP_QUOTA:
-        {
-                if (ctx && !op_errstr) {
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-                }
-                break;
-        }
-        case GD_OP_HEAL_VOLUME:
-        {
-                glusterd_add_bricks_hname_path_to_dict (ctx);
-                break;
-        }
         case GD_OP_PROFILE_VOLUME:
         {
                 if (ctx && dict_get_int32 (ctx, "count", &count)) {
                         ret = dict_set_int32 (ctx, "count", 0);
                         if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
+                                gf_log (this->name, GF_LOG_ERROR,
                                         "failed to set count in dictionary");
                         }
                 }
@@ -130,13 +111,14 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_START_BRICK:
         case GD_OP_STOP_BRICK:
         {
-                gf_log ("", GF_LOG_DEBUG, "not supported op %d", op);
+                gf_log (this->name, GF_LOG_DEBUG, "op '%s' not supported",
+                        gd_op_list[op]);
                 break;
         }
         case GD_OP_NONE:
         case GD_OP_MAX:
         {
-                gf_log ("", GF_LOG_ERROR, "invalid operation %d", op);
+                gf_log (this->name, GF_LOG_ERROR, "invalid operation");
                 break;
         }
         case GD_OP_CREATE_VOLUME:
@@ -153,10 +135,27 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_SET_VOLUME:
         case GD_OP_LIST_VOLUME:
         case GD_OP_CLEARLOCKS_VOLUME:
+        case GD_OP_HEAL_VOLUME:
+        case GD_OP_QUOTA:
         {
                 /*nothing specific to be done*/
                 break;
         }
+        case GD_OP_COPY_FILE:
+        {
+               if (ctx)
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+               break;
+        }
+        case GD_OP_SYS_EXEC:
+        {
+               if (ctx) {
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+                        ret = dict_set_str (ctx, "glusterd_workdir",
+                                            conf->workdir);
+               }
+               break;
+        }
         }
 
         rsp.op_ret = op_ret;
@@ -171,9 +170,9 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
 
         if (ctx) {
                 ret = dict_allocate_and_serialize (ctx, &rsp.dict.dict_val,
-                                                   (size_t*)&rsp.dict.dict_len);
+                                                   &rsp.dict.dict_len);
                 if (ret < 0 )
-                        gf_log (THIS->name, GF_LOG_ERROR, "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "serialize buffer");
                 else
                         free_ptr = rsp.dict.dict_val;
@@ -186,17 +185,31 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         cli_rsp = &rsp;
         xdrproc = (xdrproc_t) xdr_gf_cli_rsp;
 
-        ret = glusterd_submit_reply (req, cli_rsp, NULL, 0, NULL,
-                                     xdrproc);
+        glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                         xdrproc, ctx);
+        ret = 0;
 
-        if (free_ptr)
-                GF_FREE (free_ptr);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        GF_FREE (free_ptr);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
-glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
+glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe, fop_cbk_fn_t fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+
+        synclock_lock (&priv->big_lock);
+        ret = fn (req, iov, count, myframe);
+        synclock_unlock (&priv->big_lock);
+
+        return ret;
+}
+
+int
+__glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
                         int count, void *myframe)
 {
         gd1_mgmt_probe_rsp    rsp   = {{0},};
@@ -229,7 +242,9 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                 if (ctx->req) {
                         glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
                                                       rsp.op_errno,
-                                                      ctx->hostname, ctx->port);
+                                                      rsp.op_errstr,
+                                                      ctx->hostname, ctx->port,
+                                                      ctx->dict);
                 }
 
                 glusterd_destroy_probe_ctx (ctx);
@@ -256,7 +271,9 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                 if (ctx->req) {
                         glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
                                                       rsp.op_errno,
-                                                      ctx->hostname, ctx->port);
+                                                      rsp.op_errstr,
+                                                      ctx->hostname, ctx->port,
+                                                      ctx->dict);
                 }
 
                 glusterd_destroy_probe_ctx (ctx);
@@ -290,14 +307,22 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
         gf_log ("glusterd", GF_LOG_INFO, "Received resp to probe req");
 
 out:
-        if (rsp.hostname)
-                free (rsp.hostname);//malloced by xdr
+        free (rsp.hostname);//malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
 int
-glusterd3_1_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
+glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_probe_cbk);
+}
+
+
+int
+__glusterd_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
                             int count, void *myframe)
 {
         gd1_mgmt_friend_rsp           rsp   = {{0},};
@@ -376,21 +401,29 @@ out:
 
         if (ctx->req)//reverse probe doesn't have req
                 ret = glusterd_xfer_cli_probe_resp (ctx->req, op_ret, op_errno,
-                                                    ctx->hostname, ctx->port);
+                                                    NULL, ctx->hostname,
+                                                    ctx->port, ctx->dict);
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
         }
         if (ctx)
                 glusterd_destroy_probe_ctx (ctx);
-        if (rsp.hostname)
-                free (rsp.hostname);//malloced by xdr
+        free (rsp.hostname);//malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
 int
-glusterd3_1_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
+glusterd_friend_add_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_add_cbk);
+}
+
+int
+__glusterd_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
                                int count, void *myframe)
 {
         gd1_mgmt_friend_rsp             rsp   = {{0},};
@@ -466,8 +499,8 @@ inject:
 
 
 respond:
-        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, op_ret, op_errno,
-                                              ctx->hostname);
+        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, op_ret, op_errno, NULL,
+                                              ctx->hostname, ctx->dict);
         if (!ret && move_sm_now) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
@@ -478,37 +511,62 @@ respond:
                 glusterd_destroy_probe_ctx (ctx);
         }
 
-        if (rsp.hostname)
-                free (rsp.hostname);//malloced by xdr
+        free (rsp.hostname);//malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
+int
+glusterd_friend_remove_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_remove_cbk);
+}
+
 int32_t
-glusterd3_1_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
                               int count, void *myframe)
 {
-        int                           ret   = -1;
-        int32_t                       op_ret = 0;
-        char                          str[50] = {0,};
+        int                           ret    = -1;
+        gd1_mgmt_friend_update_rsp    rsp    = {{0}, };
+        xlator_t                      *this  = NULL;
 
         GF_ASSERT (req);
+        this = THIS;
 
         if (-1 == req->rpc_status) {
+                gf_log (this->name, GF_LOG_ERROR, "RPC Error");
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", str);
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize friend"
+                        " update repsonse");
+                goto out;
+        }
 
+        ret = 0;
 out:
+        gf_log (this->name, GF_LOG_INFO, "Received %s from uuid: %s",
+                (ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
+int
+glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_update_cbk);
+}
+
 int32_t
-glusterd3_1_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
                               int count, void *myframe)
 {
         gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
@@ -516,9 +574,18 @@ glusterd3_1_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
         int32_t                       op_ret = -1;
         glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
         glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+        glusterd_conf_t               *priv = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
         GF_ASSERT (req);
 
+        txn_id = &priv->global_txn_id;
+
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
@@ -527,7 +594,8 @@ glusterd3_1_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "response received from peer");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
                 goto out;
@@ -536,25 +604,28 @@ glusterd3_1_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         op_ret = rsp.op_ret;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received lock %s from uuid: %s", (op_ret) ? "RJT" : "ACC",
+                uuid_utoa (rsp.uuid));
 
         ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Lock response received from "
-                        "unknown peer: %s", uuid_utoa (rsp.uuid));
+                gf_log (this->name, GF_LOG_CRITICAL, "Lock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
         }
 
         if (op_ret) {
                 event_type = GD_OP_EVENT_RCVD_RJT;
                 opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
         } else {
                 event_type = GD_OP_EVENT_RCVD_ACC;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -566,16 +637,27 @@ out:
 }
 
 int32_t
-glusterd3_1_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
 {
-        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_cluster_lock_cbk);
+}
+
+static int32_t
+glusterd_vol_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        gd1_mgmt_volume_lock_rsp      rsp   = {{0},};
         int                           ret   = -1;
         int32_t                       op_ret = -1;
         glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
         glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
-
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
         if (-1 == req->rpc_status) {
@@ -584,237 +666,224 @@ glusterd3_1_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_volume_lock_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode volume lock "
+                        "response received from peer");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
                 goto out;
         }
 
-out:
         op_ret = rsp.op_ret;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+        txn_id = &rsp.txn_id;
 
-        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received volume lock %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
 
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Unlock response received from "
-                        "unknown peer %s", uuid_utoa (rsp.uuid));
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Volume lock response received "
+                        "from unknown peer: %s. Ignoring response",
+                         uuid_utoa (rsp.uuid));
+                goto out;
         }
 
         if (op_ret) {
                 event_type = GD_OP_EVENT_RCVD_RJT;
                 opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
         } else {
                 event_type = GD_OP_EVENT_RCVD_ACC;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
         }
 
+out:
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
-static int32_t
-glusterd_append_gsync_status (dict_t *dst, dict_t *src)
+int32_t
+glusterd_vol_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                       int count, void *myframe)
 {
-        int                ret = 0;
-        char               *stop_msg = NULL;
-
-        ret = dict_get_str (src, "gsync-status", &stop_msg);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (dst, "gsync-status", gf_strdup (stop_msg));
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_WARNING, "Unable to set the stop"
-                        "message in the ctx dictionary");
-                goto out;
-        }
-
-        ret = 0;
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        glusterd_vol_lock_cbk_fn);
 }
 
 static int32_t
-glusterd_append_status_dicts (dict_t *dst, dict_t *src)
+glusterd_vol_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                           int count, void *myframe)
 {
-        int              dst_count = 0;
-        int              src_count = 0;
-        int              i = 0;
-        int              ret = 0;
-        char             mst[PATH_MAX] = {0,};
-        char             slv[PATH_MAX] = {0, };
-        char             sts[PATH_MAX] = {0, };
-        char             *mst_val = NULL;
-        char             *slv_val = NULL;
-        char             *sts_val = NULL;
+        gd1_mgmt_volume_unlock_rsp    rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
-        GF_ASSERT (dst);
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
 
-        if (src == NULL)
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
                 goto out;
+        }
 
-        ret = dict_get_int32 (dst, "gsync-count", &dst_count);
-        if (ret)
-                dst_count = 0;
-
-        ret = dict_get_int32 (src, "gsync-count", &src_count);
-        if (ret || !src_count) {
-                gf_log ("", GF_LOG_DEBUG, "Source brick empty");
-                ret = 0;
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_volume_unlock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode volume unlock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
                 goto out;
         }
 
-        for (i = 1; i <= src_count; i++) {
-                snprintf (mst, sizeof(mst), "master%d", i);
-                snprintf (slv, sizeof(slv), "slave%d", i);
-                snprintf (sts, sizeof(sts), "status%d", i);
-
-                ret = dict_get_str (src, mst, &mst_val);
-                if (ret)
-                        goto out;
+        op_ret = rsp.op_ret;
 
-                ret = dict_get_str (src, slv, &slv_val);
-                if (ret)
-                        goto out;
+        txn_id = &rsp.txn_id;
 
-                ret = dict_get_str (src, sts, &sts_val);
-                if (ret)
-                        goto out;
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received volume unlock %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC",
+                uuid_utoa (rsp.uuid));
 
-                snprintf (mst, sizeof(mst), "master%d", i+dst_count);
-                snprintf (slv, sizeof(slv), "slave%d", i+dst_count);
-                snprintf (sts, sizeof(sts), "status%d", i+dst_count);
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
-                ret = dict_set_dynstr (dst, mst, gf_strdup (mst_val));
-                if (ret)
-                        goto out;
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Volume unlock response received "
+                        "from unknown peer: %s. Ignoring response",
+                        uuid_utoa (rsp.uuid));
+                goto out;
+        }
 
-                ret = dict_set_dynstr (dst, slv, gf_strdup (slv_val));
-                if (ret)
-                        goto out;
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
 
-                ret = dict_set_dynstr (dst, sts, gf_strdup (sts_val));
-                if (ret)
-                        goto out;
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
         }
 
-        ret = dict_set_int32 (dst, "gsync-count", dst_count+src_count);
-
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+out:
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
+}
 
+int32_t
+glusterd_vol_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        glusterd_vol_unlock_cbk_fn);
 }
 
-static int32_t
-glusterd_gsync_use_rsp_dict (dict_t *rsp_dict, char *op_errstr)
+int32_t
+__glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
 {
-        dict_t             *ctx = NULL;
-        int                ret = 0;
+        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+        glusterd_conf_t               *priv = NULL;
 
-        ctx = glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Operation Context is not present");
-                GF_ASSERT (0);
-        }
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (req);
 
-        if (rsp_dict) {
-                ret = glusterd_append_status_dicts (ctx, rsp_dict);
-                if (ret)
-                        goto out;
+        txn_id = &priv->global_txn_id;
 
-                ret = glusterd_append_gsync_status (ctx, rsp_dict);
-                if (ret)
-                        goto out;
-        }
-        if (strcmp ("", op_errstr)) {
-                ret = dict_set_dynstr (ctx, "errstr", gf_strdup(op_errstr));
-                if (ret)
-                        goto out;
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
         }
 
-        ret = 0;
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
-        return ret;
-}
-static int32_t
-glusterd_rb_use_rsp_dict (dict_t *rsp_dict)
-{
-        int32_t  src_port = 0;
-        int32_t  dst_port = 0;
-        int      ret      = 0;
-        dict_t  *ctx      = NULL;
-
-
-        ctx = glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Operation Context is not present");
-                GF_ASSERT (0);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
         }
 
-        if (rsp_dict) {
-                ret = dict_get_int32 (rsp_dict, "src-brick-port", &src_port);
-                if (ret == 0) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "src-brick-port=%d found", src_port);
-                }
+out:
+        op_ret = rsp.op_ret;
 
-                ret = dict_get_int32 (rsp_dict, "dst-brick-port", &dst_port);
-                if (ret == 0) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "dst-brick-port=%d found", dst_port);
-                }
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received unlock %s from uuid: %s",
+                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Unlock response received "
+                        "from unknown peer %s", uuid_utoa (rsp.uuid));
         }
 
-        if (src_port) {
-                ret = dict_set_int32 (ctx, "src-brick-port",
-                                      src_port);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not set src-brick");
-                        goto out;
-                }
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
         }
 
-        if (dst_port) {
-                ret = dict_set_int32 (ctx, "dst-brick-port",
-                                      dst_port);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not set dst-brick");
-                        goto out;
-                }
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
         }
 
-out:
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
+}
 
+int32_t
+glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_cluster_unlock_cbk);
 }
 
 int32_t
-glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         gd1_mgmt_stage_op_rsp         rsp   = {{0},};
@@ -825,22 +894,32 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
         dict_t                        *dict   = NULL;
         char                          err_str[2048] = {0};
         char                          *peer_str = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode stage "
+                        "response received from peer");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("Failed to decode stage response "
+                                        "received from peer.");
                 goto out;
         }
 
@@ -852,7 +931,7 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
                                         rsp.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize rsp-buffer to dictionary");
 			event_type = GD_OP_EVENT_RCVD_RJT;
@@ -865,15 +944,21 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
 out:
         op_ret = rsp.op_ret;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received stage %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
+
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
 
         ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Stage response received from "
-                        "unknown peer: %s", uuid_utoa (rsp.uuid));
+                gf_log (this->name, GF_LOG_CRITICAL, "Stage response received "
+                        "from unknown peer: %s. Ignoring response.",
+                        uuid_utoa (rsp.uuid));
+                goto out;
         }
 
         if (op_ret) {
@@ -886,12 +971,11 @@ out:
                                 peer_str = peerinfo->hostname;
                         else
                                 peer_str = uuid_utoa (rsp.uuid);
-                        snprintf (err_str, sizeof (err_str), "Operation failed "
-                                  "on %s", peer_str);
+                        snprintf (err_str, sizeof (err_str),
+                                  OPERRSTR_STAGE_FAIL, peer_str);
                         opinfo.op_errstr = gf_strdup (err_str);
                 }
                 if (!opinfo.op_errstr) {
-                        gf_log ("", GF_LOG_ERROR, "memory allocation failed");
                         ret = -1;
                         goto out;
                 }
@@ -901,280 +985,39 @@ out:
 
         switch (rsp.op) {
         case GD_OP_REPLACE_BRICK:
-                glusterd_rb_use_rsp_dict (dict);
+                glusterd_rb_use_rsp_dict (NULL, dict);
                 break;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
         }
 
-        if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
-                free (rsp.op_errstr); //malloced by xdr
+        free (rsp.op_errstr); //malloced by xdr
         if (dict) {
                 if (!dict->extra_stdfree && rsp.dict.dict_val)
                         free (rsp.dict.dict_val); //malloced by xdr
                 dict_unref (dict);
         } else {
-                if (rsp.dict.dict_val)
-                        free (rsp.dict.dict_val); //malloced by xdr
+                free (rsp.dict.dict_val); //malloced by xdr
         }
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
-static int32_t
-glusterd_sync_use_rsp_dict (dict_t *rsp_dict)
-{
-        int      ret      = 0;
-
-        GF_ASSERT (rsp_dict);
-
-        if (!rsp_dict) {
-                goto out;
-        }
-
-        ret = glusterd_import_friend_volumes (rsp_dict);
-out:
-        return ret;
-
-}
-
-void
-_profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
-                               void *data)
-{
-        char    new_key[256] = {0};
-        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
-        data_t  *new_value = NULL;
-        int     brick_count = 0;
-        char    brick_key[256];
-
-        if (strcmp (key, "count") == 0)
-                return;
-        sscanf (key, "%d%s", &brick_count, brick_key);
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
-        snprintf (new_key, sizeof (new_key), "%d%s",
-                  rsp_ctx->count + brick_count, brick_key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
-}
-
-int
-glusterd_profile_volume_use_rsp_dict (dict_t *rsp_dict)
-{
-        int     ret = 0;
-        glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
-        int32_t brick_count = 0;
-        int32_t count = 0;
-        dict_t  *ctx_dict = NULL;
-        glusterd_op_t   op = GD_OP_NONE;
-
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (rsp_dict, "count", &brick_count);
-        if (ret) {
-                ret = 0; //no bricks in the rsp
-                goto out;
-        }
-
-        op = glusterd_op_get_op ();
-        GF_ASSERT (GD_OP_PROFILE_VOLUME == op);
-        ctx_dict = glusterd_op_get_ctx ();
-
-        ret = dict_get_int32 (ctx_dict, "count", &count);
-        rsp_ctx.count = count;
-        rsp_ctx.dict = ctx_dict;
-        dict_foreach (rsp_dict, _profile_volume_add_friend_rsp, &rsp_ctx);
-        dict_del (ctx_dict, "count");
-        ret = dict_set_int32 (ctx_dict, "count", count + brick_count);
-out:
-        return ret;
-}
-
-void
-glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
-                                     void *data)
-{
-        glusterd_status_rsp_conv_t      *rsp_ctx = NULL;
-        data_t                          *new_value = NULL;
-        int32_t                         ret = 0;
-
-        if (strcmp (key, "count") == 0)
-                return;
-
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
-
-        ret = dict_set (rsp_ctx->dict, key, new_value);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "Unable to set key: %s in dict",
-                        key);
-
-        return;
-}
-
-int
-glusterd_volume_status_use_rsp_dict (dict_t *rsp_dict)
-{
-        int                             ret = 0;
-        glusterd_status_rsp_conv_t      rsp_ctx = {0};
-        int32_t                         brick_count = 0;
-        int32_t                         count = 0;
-        dict_t                          *ctx_dict = NULL;
-        glusterd_op_t                   op = GD_OP_NONE;
-
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (rsp_dict, "count", &brick_count);
-        if (ret) {
-                ret = 0; //no bricks in the rsp
-                goto out;
-        }
-
-        op = glusterd_op_get_op ();
-        GF_ASSERT (GD_OP_STATUS_VOLUME == op);
-        ctx_dict = glusterd_op_get_ctx (op);
-
-        ret = dict_get_int32 (ctx_dict, "count", &count);
-        rsp_ctx.count = count;
-        rsp_ctx.dict = ctx_dict;
-        dict_foreach (rsp_dict, glusterd_volume_status_add_peer_rsp, &rsp_ctx);
-
-        ret = dict_set_int32 (ctx_dict, "count", count + brick_count);
-out:
-        return ret;
-}
-
-int
-glusterd_volume_rebalance_use_rsp_dict (dict_t *rsp_dict)
-{
-        int            ret      = 0;
-        dict_t        *ctx_dict = NULL;
-        glusterd_op_t  op       = GD_OP_NONE;
-        uint64_t       value    = 0;
-        int32_t        value32  = 0;
-        char          *volname  = NULL;
-        glusterd_volinfo_t *volinfo = NULL;
-
-        GF_ASSERT (rsp_dict);
-
-        op = glusterd_op_get_op ();
-        GF_ASSERT ((GD_OP_REBALANCE == op) ||
-                   (GD_OP_DEFRAG_BRICK_VOLUME == op));
-
-        ctx_dict = glusterd_op_get_ctx (op);
-
-        if (!ctx_dict)
-                goto out;
-
-        ret = dict_get_str (ctx_dict, "volname", &volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
-                goto out;
-
-        ret = dict_get_uint64 (rsp_dict, "files", &value);
-        if (!ret) {
-                volinfo->rebalance_files += value;
-                ret = dict_set_uint64 (ctx_dict, "files",
-                                       volinfo->rebalance_files);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set the file count");
-                }
-        }
-
-        ret = dict_get_uint64 (rsp_dict, "size", &value);
-        if (!ret) {
-                volinfo->rebalance_data += value;
-                ret = dict_set_uint64 (ctx_dict, "size",
-                                       volinfo->rebalance_data);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set the size of migration");
-                }
-        }
-
-        ret = dict_get_uint64 (rsp_dict, "lookups", &value);
-        if (!ret) {
-                volinfo->lookedup_files += value;
-                ret = dict_set_uint64 (ctx_dict, "lookups",
-                                       volinfo->lookedup_files);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set lookuped file count");
-                }
-        }
-
-        ret = dict_get_int32 (rsp_dict, "status", &value32);
-        if (!ret) {
-                ret = dict_set_int32 (ctx_dict, "status", value32);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set status");
-                }
-        }
-
-out:
-        return ret;
-}
-
-void
-_heal_volume_add_peer_rsp (dict_t *peer_dict, char *key, data_t *value,
-                           void *data)
+int32_t
+glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
 {
-        int                             max_brick = 0;
-        int                             peer_max_brick = 0;
-        int                             ret = 0;
-        dict_t                          *ctx_dict = data;
-
-
-
-        ret = dict_get_int32 (ctx_dict, "count", &max_brick);
-        ret = dict_get_int32 (peer_dict, "count", &peer_max_brick);
-        if (peer_max_brick > max_brick)
-                ret = dict_set_int32 (ctx_dict, "count", peer_max_brick);
-        else
-                ret = dict_set_int32 (ctx_dict, "count", max_brick);
-        dict_del (peer_dict, "count");
-        dict_copy (peer_dict, ctx_dict);
-        return;
-}
-
-int
-glusterd_volume_heal_use_rsp_dict (dict_t *rsp_dict)
-{
-        int            ret      = 0;
-        dict_t        *ctx_dict = NULL;
-        glusterd_op_t  op       = GD_OP_NONE;
-
-        GF_ASSERT (rsp_dict);
-
-        op = glusterd_op_get_op ();
-        GF_ASSERT (GD_OP_HEAL_VOLUME == op);
-
-        ctx_dict = glusterd_op_get_ctx (op);
-
-        if (!ctx_dict)
-                goto out;
-        dict_foreach (rsp_dict, _heal_volume_add_peer_rsp, ctx_dict);
-out:
-        return ret;
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_stage_op_cbk);
 }
 
 int32_t
-glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         gd1_mgmt_commit_op_rsp         rsp   = {{0},};
@@ -1185,24 +1028,34 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
         dict_t                        *dict = NULL;
         char                          err_str[2048] = {0};
         char                          *peer_str = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
 		event_type = GD_OP_EVENT_RCVD_RJT;
                 goto out;
         }
 
         ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "response received from peer");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("Failed to decode commit response "
+                                        "received from peer.");
 		event_type = GD_OP_EVENT_RCVD_RJT;
                 goto out;
         }
@@ -1215,7 +1068,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                                         rsp.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize rsp-buffer to dictionary");
 			event_type = GD_OP_EVENT_RCVD_RJT;
@@ -1227,15 +1080,20 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
 
         op_ret = rsp.op_ret;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s",
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received commit %s from uuid: %s",
                 (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
 
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Commit response received from "
-                        "unknown peer: %s", uuid_utoa (rsp.uuid));
+                gf_log (this->name, GF_LOG_CRITICAL, "Commit response for "
+                        "'Volume %s' received from unknown peer: %s",
+                        gd_op_list[opinfo.op], uuid_utoa (rsp.uuid));
         }
 
         if (op_ret) {
@@ -1248,12 +1106,11 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                                 peer_str = peerinfo->hostname;
                         else
                                 peer_str = uuid_utoa (rsp.uuid);
-                        snprintf (err_str, sizeof (err_str), "Operation failed "
-                                  "on %s", peer_str);
+                        snprintf (err_str, sizeof (err_str),
+                                  OPERRSTR_COMMIT_FAIL, peer_str);
                         opinfo.op_errstr = gf_strdup (err_str);
                 }
                 if (!opinfo.op_errstr) {
-                        gf_log ("", GF_LOG_ERROR, "memory allocation failed");
                         ret = -1;
                         goto out;
                 }
@@ -1261,44 +1118,44 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                 event_type = GD_OP_EVENT_RCVD_ACC;
                 switch (rsp.op) {
                 case GD_OP_REPLACE_BRICK:
-                        ret = glusterd_rb_use_rsp_dict (dict);
+                        ret = glusterd_rb_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_SYNC_VOLUME:
-                        ret = glusterd_sync_use_rsp_dict (dict);
+                        ret = glusterd_sync_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_PROFILE_VOLUME:
-                        ret = glusterd_profile_volume_use_rsp_dict (dict);
+                        ret = glusterd_profile_volume_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_GSYNC_SET:
-                        ret = glusterd_gsync_use_rsp_dict (dict, rsp.op_errstr);
+                        ret = glusterd_gsync_use_rsp_dict (NULL, dict, rsp.op_errstr);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_STATUS_VOLUME:
-                        ret = glusterd_volume_status_use_rsp_dict (dict);
+                        ret = glusterd_volume_status_copy_to_op_ctx_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_REBALANCE:
                 case GD_OP_DEFRAG_BRICK_VOLUME:
-                        ret = glusterd_volume_rebalance_use_rsp_dict (dict);
+                        ret = glusterd_volume_rebalance_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
                 break;
 
                 case GD_OP_HEAL_VOLUME:
-                        ret = glusterd_volume_heal_use_rsp_dict (dict);
+                        ret = glusterd_volume_heal_use_rsp_dict (NULL, dict);
                         if (ret)
                                 goto out;
 
@@ -1310,7 +1167,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
 out:
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -1319,17 +1176,22 @@ out:
 
         if (dict)
                 dict_unref (dict);
-        if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
-                free (rsp.op_errstr); //malloced by xdr
+        free (rsp.op_errstr); //malloced by xdr
         GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
         return ret;
 }
 
-
+int32_t
+glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_commit_op_cbk);
+}
 
 int32_t
-glusterd3_1_probe (call_frame_t *frame, xlator_t *this,
-                   void *data)
+glusterd_rpc_probe (call_frame_t *frame, xlator_t *this,
+                    void *data)
 {
         gd1_mgmt_probe_req      req = {{0},};
         int                     ret = 0;
@@ -1359,26 +1221,25 @@ glusterd3_1_probe (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-        uuid_copy (req.uuid, priv->uuid);
+        uuid_copy (req.uuid, MY_UUID);
         req.hostname = gf_strdup (hostname);
         req.port = port;
 
         ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
                                        GLUSTERD_PROBE_QUERY,
-                                       NULL, this, glusterd3_1_probe_cbk,
+                                       NULL, this, glusterd_probe_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_probe_req);
 
 out:
-        if (req.hostname)
-                GF_FREE (req.hostname);
+        GF_FREE (req.hostname);
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 
 int32_t
-glusterd3_1_friend_add (call_frame_t *frame, xlator_t *this,
-                        void *data)
+glusterd_rpc_friend_add (call_frame_t *frame, xlator_t *this,
+                         void *data)
 {
         gd1_mgmt_friend_req         req      = {{0},};
         int                         ret      = 0;
@@ -1404,24 +1265,23 @@ glusterd3_1_friend_add (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-        uuid_copy (req.uuid, priv->uuid);
+        uuid_copy (req.uuid, MY_UUID);
         req.hostname = peerinfo->hostname;
         req.port = peerinfo->port;
 
         ret = dict_allocate_and_serialize (vols, &req.vols.vols_val,
-                                           (size_t *)&req.vols.vols_len);
+                                           &req.vols.vols_len);
         if (ret)
                 goto out;
 
         ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
                                        GLUSTERD_FRIEND_ADD,
-                                       NULL, this, glusterd3_1_friend_add_cbk,
+                                       NULL, this, glusterd_friend_add_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_friend_req);
 
 
 out:
-        if (req.vols.vols_val)
-                GF_FREE (req.vols.vols_val);
+        GF_FREE (req.vols.vols_val);
 
         if (vols)
                 dict_unref (vols);
@@ -1431,8 +1291,8 @@ out:
 }
 
 int32_t
-glusterd3_1_friend_remove (call_frame_t *frame, xlator_t *this,
-                           void *data)
+glusterd_rpc_friend_remove (call_frame_t *frame, xlator_t *this,
+                            void *data)
 {
         gd1_mgmt_friend_req             req = {{0},};
         int                             ret = 0;
@@ -1452,12 +1312,12 @@ glusterd3_1_friend_remove (call_frame_t *frame, xlator_t *this,
 
         peerinfo = event->peerinfo;
 
-        uuid_copy (req.uuid, priv->uuid);
+        uuid_copy (req.uuid, MY_UUID);
         req.hostname = peerinfo->hostname;
         req.port = peerinfo->port;
         ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
                                        GLUSTERD_FRIEND_REMOVE, NULL,
-                                       this, glusterd3_1_friend_remove_cbk,
+                                       this, glusterd_friend_remove_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_friend_req);
 
 out:
@@ -1467,15 +1327,13 @@ out:
 
 
 int32_t
-glusterd3_1_friend_update (call_frame_t *frame, xlator_t *this,
-                           void *data)
+glusterd_rpc_friend_update (call_frame_t *frame, xlator_t *this,
+                            void *data)
 {
         gd1_mgmt_friend_update  req         = {{0},};
         int                     ret         = 0;
         glusterd_conf_t        *priv        = NULL;
         dict_t                 *friends     = NULL;
-        char                   *dict_buf    = NULL;
-        size_t                  len         = -1;
         call_frame_t           *dummy_frame = NULL;
         glusterd_peerinfo_t    *peerinfo    = NULL;
 
@@ -1490,32 +1348,29 @@ glusterd3_1_friend_update (call_frame_t *frame, xlator_t *this,
         if (ret)
                 goto out;
 
-        ret = dict_allocate_and_serialize (friends, &dict_buf, (size_t *)&len);
+        ret = dict_allocate_and_serialize (friends, &req.friends.friends_val,
+                                           &req.friends.friends_len);
         if (ret)
                 goto out;
 
-        req.friends.friends_val = dict_buf;
-        req.friends.friends_len = len;
-
-        uuid_copy (req.uuid, priv->uuid);
+        uuid_copy (req.uuid, MY_UUID);
 
         dummy_frame = create_frame (this, this->ctx->pool);
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->peer,
                                        GLUSTERD_FRIEND_UPDATE, NULL,
-                                       this, glusterd3_1_friend_update_cbk,
+                                       this, glusterd_friend_update_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_friend_update);
 
 out:
-        if (req.friends.friends_val)
-                GF_FREE (req.friends.friends_val);
+        GF_FREE (req.friends.friends_val);
 
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_cluster_lock (call_frame_t *frame, xlator_t *this,
+glusterd_cluster_lock (call_frame_t *frame, xlator_t *this,
                           void *data)
 {
         gd1_mgmt_cluster_lock_req       req = {{0},};
@@ -1541,15 +1396,146 @@ glusterd3_1_cluster_lock (call_frame_t *frame, xlator_t *this,
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_LOCK,
                                        NULL,
-                                       this, glusterd3_1_cluster_lock_cbk,
+                                       this, glusterd_cluster_lock_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_cluster_unlock (call_frame_t *frame, xlator_t *this,
+glusterd_vol_lock (call_frame_t *frame, xlator_t *this,
+                   void *data)
+{
+        gd1_mgmt_volume_lock_req         req         = {{0},};
+        int                              ret         = -1;
+        glusterd_peerinfo_t             *peerinfo    = NULL;
+        glusterd_conf_t                 *priv        = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict        = NULL;
+        uuid_t                          *txn_id      = NULL;
+
+        if (!this)
+                goto out;
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+        ret = dict_get_bin (dict, "transaction_id",
+                            (void **)&txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to get transaction id.");
+                goto out;
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+                uuid_copy (req.txn_id, *txn_id);
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt_v3,
+                                       GLUSTERD_MGMT_V3_VOLUME_LOCK, NULL,
+                                       this, glusterd_vol_lock_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_volume_lock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_vol_unlock (call_frame_t *frame, xlator_t *this,
+                   void *data)
+{
+        gd1_mgmt_volume_unlock_req       req         = {{0},};
+        int                              ret         = -1;
+        glusterd_peerinfo_t             *peerinfo    = NULL;
+        glusterd_conf_t                 *priv        = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict        = NULL;
+        uuid_t                          *txn_id      = NULL;
+
+        if (!this)
+                goto out;
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+        ret = dict_get_bin (dict, "transaction_id",
+                            (void **)&txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to get transaction id.");
+                goto out;
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+                uuid_copy (req.txn_id, *txn_id);
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt_v3,
+                                       GLUSTERD_MGMT_V3_VOLUME_UNLOCK, NULL,
+                                       this, glusterd_vol_unlock_cbk,
+                                       (xdrproc_t)
+                                       xdr_gd1_mgmt_volume_unlock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_cluster_unlock (call_frame_t *frame, xlator_t *this,
                             void *data)
 {
         gd1_mgmt_cluster_lock_req       req = {{0},};
@@ -1575,15 +1561,15 @@ glusterd3_1_cluster_unlock (call_frame_t *frame, xlator_t *this,
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_UNLOCK,
                                        NULL,
-                                       this, glusterd3_1_cluster_unlock_cbk,
+                                       this, glusterd_cluster_unlock_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_stage_op (call_frame_t *frame, xlator_t *this,
+glusterd_stage_op (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         gd1_mgmt_stage_op_req           req = {{0,},};
@@ -1614,9 +1600,12 @@ glusterd3_1_stage_op (call_frame_t *frame, xlator_t *this,
         req.op = glusterd_op_get_op ();
 
         ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
-                                           (size_t *)&req.buf.buf_len);
-        if (ret)
+                                           &req.buf.buf_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
                 goto out;
+        }
 
 
         dummy_frame = create_frame (this, this->ctx->pool);
@@ -1626,19 +1615,19 @@ glusterd3_1_stage_op (call_frame_t *frame, xlator_t *this,
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->mgmt, GLUSTERD_MGMT_STAGE_OP,
                                        NULL,
-                                       this, glusterd3_1_stage_op_cbk,
+                                       this, glusterd_stage_op_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
 
 out:
         if ((_gf_true == is_alloc) && req.buf.buf_val)
                 GF_FREE (req.buf.buf_val);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_commit_op (call_frame_t *frame, xlator_t *this,
+glusterd_commit_op (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         gd1_mgmt_commit_op_req  req         = {{0,},};
@@ -1668,9 +1657,12 @@ glusterd3_1_commit_op (call_frame_t *frame, xlator_t *this,
         req.op = glusterd_op_get_op ();
 
         ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
-                                           (size_t *)&req.buf.buf_len);
-        if (ret)
+                                           &req.buf.buf_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict to "
+                        "request buffer");
                 goto out;
+        }
 
         dummy_frame = create_frame (this, this->ctx->pool);
         if (!dummy_frame)
@@ -1679,19 +1671,19 @@ glusterd3_1_commit_op (call_frame_t *frame, xlator_t *this,
         ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
                                        peerinfo->mgmt, GLUSTERD_MGMT_COMMIT_OP,
                                        NULL,
-                                       this, glusterd3_1_commit_op_cbk,
+                                       this, glusterd_commit_op_cbk,
                                        (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
 
 out:
         if ((_gf_true == is_alloc) && req.buf.buf_val)
                 GF_FREE (req.buf.buf_val);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                           int count, void *myframe)
 {
         gd1_mgmt_brick_op_rsp         rsp   = {0};
@@ -1704,22 +1696,34 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
         int                           index = 0;
         glusterd_req_ctx_t            *req_ctx = NULL;
         glusterd_pending_node_t       *node = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+        glusterd_conf_t               *priv = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
         GF_ASSERT (req);
+
+        txn_id = &priv->global_txn_id;
         frame = myframe;
         req_ctx = frame->local;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "error";
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
 		event_type = GD_OP_EVENT_RCVD_RJT;
                 goto out;
         }
 
         ret =  xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode brick op "
+                        "response received");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
                 rsp.op_errstr = strdup ("Unable to decode brick op response");
@@ -1735,8 +1739,7 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                                         rsp.output.output_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
                                 "unserialize rsp-buffer to dictionary");
 			event_type = GD_OP_EVENT_RCVD_RJT;
                         goto out;
@@ -1753,7 +1756,7 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                 index = node->index;
                 ret = dict_set_int32 (dict, "index", index);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "Error setting index on brick status rsp dict");
                         rsp.op_ret = -1;
                         event_type = GD_OP_EVENT_RCVD_RJT;
@@ -1761,6 +1764,11 @@ glusterd3_1_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                 }
         }
 out:
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx), gf_gld_mt_brick_rsp_ctx_t);
         GF_ASSERT (ev_ctx);
         if (op_ret) {
@@ -1773,7 +1781,7 @@ out:
         ev_ctx->pending_node = frame->cookie;
         ev_ctx->rsp_dict  = dict;
         ev_ctx->commit_ctx = frame->local;
-        ret = glusterd_op_sm_inject_event (event_type, ev_ctx);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, ev_ctx);
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
@@ -1781,16 +1789,24 @@ out:
 
         if (ret && dict)
                 dict_unref (dict);
-        if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
-                free (rsp.op_errstr); //malloced by xdr
+        free (rsp.op_errstr); //malloced by xdr
         GLUSTERD_STACK_DESTROY (frame);
         return ret;
 }
 
 int32_t
-glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
-                      void *data)
+glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_brick_op_cbk);
+}
+
+int32_t
+glusterd_brick_op (call_frame_t *frame, xlator_t *this,
+                   void *data)
 {
+
         gd1_mgmt_brick_op_req           *req = NULL;
         int                             ret = 0;
         glusterd_conf_t                 *priv = NULL;
@@ -1800,6 +1816,8 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
         glusterd_pending_node_t         *pending_node;
         glusterd_req_ctx_t              *req_ctx = NULL;
         struct rpc_clnt                 *rpc = NULL;
+        dict_t                          *op_ctx = NULL;
+        uuid_t                          *txn_id = NULL;
 
         if (!this) {
                 ret = -1;
@@ -1808,34 +1826,78 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
         priv = this->private;
         GF_ASSERT (priv);
 
+        txn_id = &priv->global_txn_id;
+
         req_ctx = data;
         GF_ASSERT (req_ctx);
         INIT_LIST_HEAD (&opinfo.pending_bricks);
-        ret = glusterd_op_bricks_select (req_ctx->op, req_ctx->dict, &op_errstr);
+        ret = glusterd_op_bricks_select (req_ctx->op, req_ctx->dict, &op_errstr,
+                                         &opinfo.pending_bricks, NULL);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Brick Op failed");
+                gf_log (this->name, GF_LOG_ERROR, "Failed to select bricks "
+                        "while performing brick op during 'Volume %s'",
+                        gd_op_list[opinfo.op]);
                 opinfo.op_errstr = op_errstr;
                 goto out;
         }
 
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         list_for_each_entry (pending_node, &opinfo.pending_bricks, list) {
                 dummy_frame = create_frame (this, this->ctx->pool);
                 if (!dummy_frame)
                         continue;
 
-                ret = glusterd_brick_op_build_payload (req_ctx->op,
-                                                       pending_node->node,
-                                                       (gd1_mgmt_brick_op_req **)&req,
-                                                       req_ctx->dict);
-                if (ret)
-                        goto out;
+                if ((pending_node->type == GD_NODE_NFS) ||
+                    (pending_node->type == GD_NODE_QUOTAD) ||
+                    ((pending_node->type == GD_NODE_SHD) &&
+                     (req_ctx->op == GD_OP_STATUS_VOLUME)))
+                        ret = glusterd_node_op_build_payload
+                                (req_ctx->op,
+                                 (gd1_mgmt_brick_op_req **)&req,
+                                 req_ctx->dict);
+                else {
+                        ret = glusterd_brick_op_build_payload
+                                (req_ctx->op, pending_node->node,
+                                 (gd1_mgmt_brick_op_req **)&req,
+                                 req_ctx->dict);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "build brick op payload during "
+                                        "'Volume %s'", gd_op_list[req_ctx->op]);
+                                goto out;
+                        }
+                }
 
                 dummy_frame->local = data;
                 dummy_frame->cookie = pending_node;
 
                 rpc = glusterd_pending_node_get_rpc (pending_node);
                 if (!rpc) {
+                        if (pending_node->type == GD_NODE_REBALANCE) {
+                                opinfo.brick_pending_count = 0;
+                                ret = 0;
+                                if (req) {
+                                        GF_FREE (req->input.input_val);
+                                        GF_FREE (req);
+                                        req = NULL;
+                                }
+                                GLUSTERD_STACK_DESTROY (dummy_frame);
+
+                                op_ctx = glusterd_op_get_ctx ();
+                                if (!op_ctx)
+                                        goto out;
+                                glusterd_defrag_volume_node_rsp (req_ctx->dict,
+                                                                 NULL, op_ctx);
+
+                                goto out;
+                        }
+
                         ret = -1;
                         gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
                                 "due to rpc failure.");
@@ -1845,11 +1907,10 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
                 ret = glusterd_submit_request (rpc, req, dummy_frame,
                                                priv->gfs_mgmt,
                                                req->op, NULL,
-                                               this, glusterd3_1_brick_op_cbk,
+                                               this, glusterd_brick_op_cbk,
                                                (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
                 if (req) {
-                        if (req->input.input_val)
-                                GF_FREE (req->input.input_val);
+                        GF_FREE (req->input.input_val);
                         GF_FREE (req);
                         req = NULL;
                 }
@@ -1857,38 +1918,46 @@ glusterd3_1_brick_op (call_frame_t *frame, xlator_t *this,
                         pending_bricks++;
         }
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Sent op req to %d bricks",
+        gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for operation "
+                "'Volume %s' to %d bricks", gd_op_list[req_ctx->op],
                 pending_bricks);
         opinfo.brick_pending_count = pending_bricks;
 
 out:
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, data);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             txn_id, data);
                 opinfo.op_ret = ret;
         }
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 struct rpc_clnt_procedure gd_brick_actors[GLUSTERD_BRICK_MAXVALUE] = {
         [GLUSTERD_BRICK_NULL] = {"NULL", NULL },
-        [GLUSTERD_BRICK_OP]   = {"BRICK_OP", glusterd3_1_brick_op},
+        [GLUSTERD_BRICK_OP]   = {"BRICK_OP", glusterd_brick_op},
 };
 
 struct rpc_clnt_procedure gd_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
         [GLUSTERD_FRIEND_NULL]   = {"NULL", NULL },
-        [GLUSTERD_PROBE_QUERY]   = {"PROBE_QUERY", glusterd3_1_probe},
-        [GLUSTERD_FRIEND_ADD]    = {"FRIEND_ADD", glusterd3_1_friend_add},
-        [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd3_1_friend_remove},
-        [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd3_1_friend_update},
+        [GLUSTERD_PROBE_QUERY]   = {"PROBE_QUERY", glusterd_rpc_probe},
+        [GLUSTERD_FRIEND_ADD]    = {"FRIEND_ADD", glusterd_rpc_friend_add},
+        [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd_rpc_friend_remove},
+        [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_rpc_friend_update},
 };
 
 struct rpc_clnt_procedure gd_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
         [GLUSTERD_MGMT_NULL]           = {"NULL", NULL },
-        [GLUSTERD_MGMT_CLUSTER_LOCK]   = {"CLUSTER_LOCK", glusterd3_1_cluster_lock},
-        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK", glusterd3_1_cluster_unlock},
-        [GLUSTERD_MGMT_STAGE_OP]       = {"STAGE_OP", glusterd3_1_stage_op},
-        [GLUSTERD_MGMT_COMMIT_OP]      = {"COMMIT_OP", glusterd3_1_commit_op},
+        [GLUSTERD_MGMT_CLUSTER_LOCK]   = {"CLUSTER_LOCK", glusterd_cluster_lock},
+        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK", glusterd_cluster_unlock},
+        [GLUSTERD_MGMT_STAGE_OP]       = {"STAGE_OP", glusterd_stage_op},
+        [GLUSTERD_MGMT_COMMIT_OP]      = {"COMMIT_OP", glusterd_commit_op},
+};
+
+struct rpc_clnt_procedure gd_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
+        [GLUSTERD_MGMT_V3_NULL]           = {"NULL", NULL },
+        [GLUSTERD_MGMT_V3_VOLUME_LOCK]    = {"VOLUME_LOCK", glusterd_vol_lock},
+        [GLUSTERD_MGMT_V3_VOLUME_UNLOCK]  = {"VOLUME_UNLOCK", glusterd_vol_unlock},
 };
 
 struct rpc_clnt_program gd_mgmt_prog = {
@@ -1915,3 +1984,10 @@ struct rpc_clnt_program gd_peer_prog = {
         .numproc   = GLUSTERD_FRIEND_MAXVALUE,
 };
 
+struct rpc_clnt_program gd_mgmt_v3_prog = {
+        .progname  = "glusterd mgmt v3",
+        .prognum   = GD_MGMT_PROGRAM,
+        .progver   = GD_MGMT_V3_VERSION,
+        .proctable = gd_mgmt_v3_actors,
+        .numproc   = GLUSTERD_MGMT_V3_MAXVALUE,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index 259c247f4..c671edf68 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -100,8 +90,7 @@ glusterd_destroy_probe_ctx (glusterd_probe_ctx_t *ctx)
         if (!ctx)
                 return;
 
-        if (ctx->hostname)
-                GF_FREE (ctx->hostname);
+        GF_FREE (ctx->hostname);
         GF_FREE (ctx);
 }
 
@@ -113,8 +102,7 @@ glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx)
 
         if (ctx->vols)
                 dict_unref (ctx->vols);
-        if (ctx->hostname)
-                GF_FREE (ctx->hostname);
+        GF_FREE (ctx->hostname);
         GF_FREE (ctx);
 }
 
@@ -123,8 +111,7 @@ glusterd_destroy_friend_update_ctx (glusterd_friend_update_ctx_t *ctx)
 {
         if (!ctx)
                 return;
-        if (ctx->hostname)
-                GF_FREE (ctx->hostname);
+        GF_FREE (ctx->hostname);
         GF_FREE (ctx);
 }
 
@@ -257,12 +244,9 @@ glusterd_ac_reverse_probe_begin (glusterd_friend_sm_event_t *event, void *ctx)
 
 out:
         if (ret) {
-                if (new_event)
-                        GF_FREE (new_event);
-                if (new_ev_ctx->hostname)
-                        GF_FREE (new_ev_ctx->hostname);
-                if (new_ev_ctx)
-                        GF_FREE (new_ev_ctx);
+                GF_FREE (new_event);
+                GF_FREE (new_ev_ctx->hostname);
+                GF_FREE (new_ev_ctx);
         }
         gf_log ("", GF_LOG_DEBUG, "returning with %d", ret);
         return ret;
@@ -415,7 +399,9 @@ glusterd_ac_send_friend_remove_req (glusterd_friend_sm_event_t *event,
 
                 if (ctx)
                         ret = glusterd_xfer_cli_deprobe_resp (ctx->req, ret, 0,
-                                                              ctx->hostname);
+                                                              NULL,
+                                                              ctx->hostname,
+                                                              ctx->dict);
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
 
@@ -444,22 +430,36 @@ out:
         return ret;
 }
 
+static gf_boolean_t
+glusterd_should_update_peer (glusterd_peerinfo_t *peerinfo,
+                             glusterd_peerinfo_t *cur_peerinfo)
+{
+        gf_boolean_t is_valid = _gf_false;
+
+        if ((peerinfo == cur_peerinfo) ||
+            (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED))
+                is_valid = _gf_true;
+
+        return is_valid;
+}
+
 static int
 glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                           ret         = 0;
-        glusterd_peerinfo_t          *peerinfo    = NULL;
-        rpc_clnt_procedure_t         *proc        = NULL;
-        xlator_t                     *this        = NULL;
-        glusterd_friend_update_ctx_t  ev_ctx      = {{0}};
-        glusterd_conf_t              *priv        = NULL;
-        dict_t                       *friends     = NULL;
-        char                          key[100]    = {0,};
-        char                         *dup_buf     = NULL;
-        int32_t                       count       = 0;
+        int                           ret               = 0;
+        glusterd_peerinfo_t          *cur_peerinfo      = NULL;
+        glusterd_peerinfo_t          *peerinfo          = NULL;
+        rpc_clnt_procedure_t         *proc              = NULL;
+        xlator_t                     *this              = NULL;
+        glusterd_friend_update_ctx_t  ev_ctx            = {{0}};
+        glusterd_conf_t              *priv              = NULL;
+        dict_t                       *friends           = NULL;
+        char                          key[100]          = {0,};
+        char                         *dup_buf           = NULL;
+        int32_t                       count             = 0;
 
         GF_ASSERT (event);
-        peerinfo = event->peerinfo;
+        cur_peerinfo = event->peerinfo;
 
         this = THIS;
         priv = this->private;
@@ -478,6 +478,9 @@ glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
                 goto out;
 
         list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
+                        continue;
+
                 count++;
                 snprintf (key, sizeof (key), "friend%d.uuid", count);
                 dup_buf = gf_strdup (uuid_utoa (peerinfo->uuid));
@@ -500,6 +503,9 @@ glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
                 if (!peerinfo->connected || !peerinfo->peer)
                         continue;
 
+                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
+                        continue;
+
                 ret = dict_set_static_ptr (friends, "peerinfo", peerinfo);
                 if (ret) {
                         gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
@@ -533,7 +539,7 @@ glusterd_peer_detach_cleanup (glusterd_conf_t *priv)
         list_for_each_entry_safe (volinfo,tmp_volinfo,
                                   &priv->volumes, vol_list) {
                 if (!glusterd_friend_contains_vol_bricks (volinfo,
-                                                          priv->uuid)) {
+                                                          MY_UUID)) {
                         gf_log (THIS->name, GF_LOG_INFO,
                                 "Deleting stale volume %s", volinfo->volname);
                         ret = glusterd_delete_volume (volinfo);
@@ -645,7 +651,8 @@ glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
         uuid_copy (peerinfo->uuid, ev_ctx->uuid);
 
         //Build comparison logic here.
-        ret = glusterd_compare_friend_data (ev_ctx->vols, &status);
+        ret = glusterd_compare_friend_data (ev_ctx->vols, &status,
+                                            peerinfo->hostname);
         if (ret)
                 goto out;
 
@@ -682,7 +689,8 @@ glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
         glusterd_friend_sm_inject_event (new_event);
 
         ret = glusterd_xfer_friend_add_resp (ev_ctx->req, ev_ctx->hostname,
-                                             ev_ctx->port, op_ret, op_errno);
+                                             peerinfo->hostname, ev_ctx->port,
+                                             op_ret, op_errno);
 
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
@@ -954,6 +962,27 @@ glusterd_destroy_friend_event_context (glusterd_friend_sm_event_t *event)
         }
 }
 
+gf_boolean_t
+gd_does_peer_affect_quorum (glusterd_friend_sm_state_t old_state,
+                            glusterd_friend_sm_event_type_t event_type,
+                            glusterd_peerinfo_t *peerinfo)
+{
+        gf_boolean_t    affects = _gf_false;
+
+        //When glusterd comes up with friends in BEFRIENDED state in store,
+        //wait until compare-data happens.
+        if ((old_state == GD_FRIEND_STATE_BEFRIENDED) &&
+            (event_type != GD_FRIEND_EVENT_RCVD_ACC) &&
+            (event_type != GD_FRIEND_EVENT_LOCAL_ACC))
+                goto out;
+        if ((peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)
+            && peerinfo->connected) {
+                affects = _gf_true;
+        }
+out:
+        return affects;
+}
+
 int
 glusterd_friend_sm ()
 {
@@ -965,6 +994,8 @@ glusterd_friend_sm ()
         glusterd_peerinfo_t             *peerinfo   = NULL;
         glusterd_friend_sm_event_type_t  event_type = 0;
         gf_boolean_t                     is_await_conn = _gf_false;
+        gf_boolean_t                     quorum_action = _gf_false;
+        glusterd_friend_sm_state_t       old_state = GD_FRIEND_STATE_DEFAULT;
 
         while (!list_empty (&gd_friend_sm_queue)) {
                 list_for_each_entry_safe (event, tmp, &gd_friend_sm_queue, list) {
@@ -984,6 +1015,7 @@ glusterd_friend_sm ()
                                 glusterd_friend_sm_event_name_get (event_type));
 
 
+                        old_state = peerinfo->state.state;
                         state = glusterd_friend_state_table[peerinfo->state.state];
 
                         GF_ASSERT (state);
@@ -1024,6 +1056,15 @@ glusterd_friend_sm ()
                                 goto out;
                         }
 
+                        if (gd_does_peer_affect_quorum (old_state, event_type,
+                                                        peerinfo)) {
+                                peerinfo->quorum_contrib = QUORUM_UP;
+                                if (peerinfo->quorum_action) {
+                                        peerinfo->quorum_action = _gf_false;
+                                        quorum_action = _gf_true;
+                                }
+                        }
+
                         ret = glusterd_store_peerinfo (peerinfo);
 
                         glusterd_destroy_friend_event_context (event);
@@ -1037,6 +1078,25 @@ glusterd_friend_sm ()
 
         ret = 0;
 out:
+        if (quorum_action) {
+            /* When glusterd is restarted, it needs to wait until the 'friends' view
+             * of the volumes settle, before it starts any of the internal daemons.
+             *
+             * Every friend that was part of the cluster, would send its
+             * cluster-view, 'our' way. For every friend, who belongs to
+             * a partition which has a different cluster-view from our
+             * partition, we may update our cluster-view. For subsequent
+             * friends from that partition would agree with us, if the first
+             * friend wasn't rejected. For every first friend, whom we agreed with,
+             * we would need to start internal daemons/bricks belonging to the
+             * new volumes.
+             * glusterd_spawn_daemons calls functions that are idempotent. ie,
+             * the functions spawn process(es) only if they are not started yet.
+             *
+             * */
+                glusterd_spawn_daemons (NULL);
+                glusterd_do_quorum_action ();
+        }
         return ret;
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index 7baaf0236..b9bedbe69 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_SM_H_
 #define _GLUSTERD_SM_H_
 
@@ -37,14 +27,21 @@
 #include "byte-order.h"
 //#include "glusterd.h"
 #include "rpcsvc.h"
-
-struct glusterd_store_handle_ {
-        char    *path;
-        int     fd;
-        FILE    *read;
-};
-
-typedef struct glusterd_store_handle_  glusterd_store_handle_t;
+#include "store.h"
+
+typedef enum gd_quorum_contribution_ {
+        QUORUM_NONE,
+        QUORUM_WAITING,
+        QUORUM_DOWN,
+        QUORUM_UP
+} gd_quorum_contrib_t;
+
+typedef enum gd_quorum_status_ {
+        QUORUM_UNKNOWN,
+        QUORUM_NOT_APPLICABLE,
+        QUORUM_MEETS,
+        QUORUM_DOES_NOT_MEET
+} gd_quorum_status_t;
 
 typedef enum glusterd_friend_sm_state_ {
         GD_FRIEND_STATE_DEFAULT = 0,
@@ -89,7 +86,9 @@ typedef struct glusterd_sm_tr_log_ {
 
 struct glusterd_peerinfo_ {
         uuid_t                          uuid;
-        char                            uuid_str[50];
+        char                            uuid_str[50]; /* Retrieve this using
+                                                       * gd_peer_uuid_str ()
+                                                       */
         glusterd_peer_state_info_t      state;
         char                            *hostname;
         int                             port;
@@ -98,9 +97,13 @@ struct glusterd_peerinfo_ {
         struct rpc_clnt                 *rpc;
         rpc_clnt_prog_t                 *mgmt;
         rpc_clnt_prog_t                 *peer;
+        rpc_clnt_prog_t                 *mgmt_v3;
         int                             connected;
-        glusterd_store_handle_t         *shandle;
+        gf_store_handle_t         *shandle;
         glusterd_sm_tr_log_t            sm_log;
+        gf_boolean_t                    quorum_action;
+        gd_quorum_contrib_t             quorum_contrib;
+        gf_boolean_t                    locked;
 };
 
 typedef struct glusterd_peerinfo_ glusterd_peerinfo_t;
@@ -114,11 +117,13 @@ typedef enum glusterd_ev_gen_mode_ {
 typedef struct glusterd_peer_ctx_args_ {
         rpcsvc_request_t        *req;
         glusterd_ev_gen_mode_t  mode;
+        dict_t                  *dict;
 } glusterd_peerctx_args_t;
 
 typedef struct glusterd_peer_ctx_ {
         glusterd_peerctx_args_t        args;
         glusterd_peerinfo_t            *peerinfo;
+        char                           *errstr;
 } glusterd_peerctx_t;
 
 typedef enum glusterd_friend_sm_event_type_ {
@@ -179,6 +184,7 @@ typedef struct glusterd_probe_ctx_ {
         char                    *hostname;
         rpcsvc_request_t        *req;
         int                      port;
+        dict_t                  *dict;
 } glusterd_probe_ctx_t;
 int
 glusterd_friend_sm_new_event (glusterd_friend_sm_event_type_t event_type,
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 56b75ec90..d5bfeda3f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1,21 +1,13 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
+
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -42,6 +34,8 @@
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-hooks.h"
+#include "store.h"
 #include "glusterd-store.h"
 
 #include "rpc-clnt.h"
@@ -51,101 +45,7 @@
 #include <inttypes.h>
 #include <dirent.h>
 
-static int32_t
-glusterd_store_mkdir (char *path)
-{
-        int32_t     ret = -1;
-
-        ret = mkdir (path, 0777);
-
-        if ((-1 == ret) && (EEXIST != errno)) {
-                gf_log ("", GF_LOG_ERROR, "mkdir() failed on path %s,"
-                        "errno: %s", path, strerror (errno));
-        } else {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-int32_t
-glusterd_store_handle_create_on_absence (glusterd_store_handle_t **shandle,
-                                         char *path)
-{
-        GF_ASSERT (shandle);
-        int32_t     ret = 0;
-
-        if (*shandle == NULL) {
-                ret = glusterd_store_handle_new (path, shandle);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to create store"
-                                " handle for path: %s", path);
-                }
-        }
-        return ret;
-}
-
-int32_t
-glusterd_store_mkstemp (glusterd_store_handle_t *shandle)
-{
-        int     fd = -1;
-        char    tmppath[PATH_MAX] = {0,};
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->path);
-
-        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
-        fd = open (tmppath, O_RDWR | O_CREAT | O_TRUNC, 0600);
-        if (fd <= 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to open %s, "
-                        "error: %s", tmppath, strerror (errno));
-        }
-
-        return fd;
-}
-
-int32_t
-glusterd_store_rename_tmppath (glusterd_store_handle_t *shandle)
-{
-        int32_t         ret = -1;
-        char            tmppath[PATH_MAX] = {0,};
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->path);
-
-        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
-        ret = rename (tmppath, shandle->path);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to mv %s to %s, "
-                        "error: %s", tmppath, shandle->path, strerror (errno));
-        }
-
-        return ret;
-}
-
-int32_t
-glusterd_store_unlink_tmppath (glusterd_store_handle_t *shandle)
-{
-        int32_t         ret = -1;
-        char            tmppath[PATH_MAX] = {0,};
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->path);
-
-        snprintf (tmppath, sizeof (tmppath), "%s.tmp", shandle->path);
-        ret = unlink (tmppath);
-        if (ret && (errno != ENOENT)) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to mv %s to %s, "
-                        "error: %s", tmppath, shandle->path, strerror (errno));
-        } else {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-static void
+void
 glusterd_replace_slash_with_hipen (char *str)
 {
         char                    *ptr = NULL;
@@ -171,7 +71,7 @@ glusterd_store_create_brick_dir (glusterd_volinfo_t *volinfo)
         GF_ASSERT (priv);
 
         GLUSTERD_GET_BRICK_DIR (brickdirpath, volinfo, priv);
-        ret = glusterd_store_mkdir (brickdirpath);
+        ret = gf_store_mkdir (brickdirpath);
 
         return ret;
 }
@@ -232,20 +132,31 @@ glusterd_store_is_valid_brickpath (char *volname, char *brick)
         glusterd_brickinfo_t    *brickinfo = NULL;
         glusterd_volinfo_t      *volinfo = NULL;
         int32_t                 ret = 0;
+        size_t                  volname_len = strlen (volname);
+        xlator_t                *this = NULL;
 
-        ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "brick path validation failed");
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create brick "
+                        "info for brick %s", brick);
                 ret = 0;
                 goto out;
         }
         ret = glusterd_volinfo_new (&volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "brick path validation failed");
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create volinfo");
                 ret = 0;
                 goto out;
         }
-        strncpy (volinfo->volname, volname, sizeof (volinfo->volname));
+        if (volname_len >= sizeof (volinfo->volname)) {
+                gf_log (this->name, GF_LOG_WARNING, "volume name too long");
+                ret = 0;
+                goto out;
+        }
+        memcpy (volinfo->volname, volname, volname_len+1);
         glusterd_store_brickinfopath_set (volinfo, brickinfo, brickpath,
                                                 sizeof (brickpath));
 
@@ -255,7 +166,7 @@ out:
         if (brickinfo)
                 glusterd_brickinfo_delete (brickinfo);
         if (volinfo)
-                glusterd_volinfo_delete (volinfo);
+                glusterd_volinfo_unref (volinfo);
 
         return ret;
 }
@@ -273,7 +184,11 @@ glusterd_store_volinfo_brick_fname_write (int vol_fd,
                   brick_count);
         glusterd_store_brickinfofname_set (brickinfo, brickfname,
                                         sizeof (brickfname));
-        ret = glusterd_store_save_value (vol_fd, key, brickfname);
+        ret = gf_store_save_value (vol_fd, key, brickfname);
+        if (ret)
+                goto out;
+
+out:
         return ret;
 }
 
@@ -288,9 +203,9 @@ glusterd_store_create_brick_shandle_on_absence (glusterd_volinfo_t *volinfo,
         GF_ASSERT (brickinfo);
 
         glusterd_store_brickinfopath_set (volinfo, brickinfo, brickpath,
-                                      sizeof (brickpath));
-        ret = glusterd_store_handle_create_on_absence (&brickinfo->shandle,
-                                                       brickpath);
+                                          sizeof (brickpath));
+        ret = gf_store_handle_create_on_absence (&brickinfo->shandle,
+                                                 brickpath);
         return ret;
 }
 
@@ -303,30 +218,41 @@ glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo)
         GF_ASSERT (brickinfo);
         GF_ASSERT (fd > 0);
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
-                                         brickinfo->hostname);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
+                                   brickinfo->hostname);
         if (ret)
                 goto out;
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PATH,
-                                         brickinfo->path);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PATH,
+                                   brickinfo->path);
         if (ret)
                 goto out;
 
         snprintf (value, sizeof(value), "%d", brickinfo->port);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PORT,
-                                         value);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PORT, value);
 
         snprintf (value, sizeof(value), "%d", brickinfo->rdma_port);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
-                                         value);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                   value);
 
         snprintf (value, sizeof(value), "%d", brickinfo->decommissioned);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
-                                         value);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                                   value);
+        if (ret)
+                goto out;
 
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_ID,
+                                   brickinfo->brick_id);
+        if (ret)
+                goto out;
+
+        if (!brickinfo->vg[0])
+                goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                         brickinfo->vg);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -337,7 +263,7 @@ glusterd_store_perform_brick_store (glusterd_brickinfo_t *brickinfo)
         int32_t                     ret = -1;
         GF_ASSERT (brickinfo);
 
-        fd = glusterd_store_mkstemp (brickinfo->shandle);
+        fd = gf_store_mkstemp (brickinfo->shandle);
         if (fd <= 0) {
                 ret = -1;
                 goto out;
@@ -347,13 +273,12 @@ glusterd_store_perform_brick_store (glusterd_brickinfo_t *brickinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_store_rename_tmppath (brickinfo->shandle);
 out:
         if (ret && (fd > 0))
-                glusterd_store_unlink_tmppath (brickinfo->shandle);
+                gf_store_unlink_tmppath (brickinfo->shandle);
         if (fd > 0)
                 close (fd);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -383,30 +308,27 @@ glusterd_store_brickinfo (glusterd_volinfo_t *volinfo,
 
         ret = glusterd_store_perform_brick_store (brickinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
 int32_t
-glusterd_store_delete_brick (glusterd_volinfo_t *volinfo,
-                             glusterd_brickinfo_t *brickinfo)
+glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo, char *delete_path)
 {
         int32_t                 ret = -1;
         glusterd_conf_t         *priv = NULL;
-        char                    path[PATH_MAX] = {0,};
         char                    brickpath[PATH_MAX] = {0,};
         char                    *ptr = NULL;
         char                    *tmppath = NULL;
+        xlator_t                *this = NULL;
 
-        GF_ASSERT (volinfo);
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (brickinfo);
 
-        priv = THIS->private;
-
+        priv = this->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_BRICK_DIR (path, volinfo, priv);
-
         tmppath = gf_strdup (brickinfo->path);
 
         ptr = strchr (tmppath, '/');
@@ -416,16 +338,17 @@ glusterd_store_delete_brick (glusterd_volinfo_t *volinfo,
                 ptr = strchr (tmppath, '/');
         }
 
-        snprintf (brickpath, sizeof (brickpath), "%s/%s:%s",
-                  path, brickinfo->hostname, tmppath);
+        snprintf (brickpath, sizeof (brickpath),
+                  "%s/"GLUSTERD_BRICK_INFO_DIR"/%s:%s", delete_path,
+                  brickinfo->hostname, tmppath);
 
         GF_FREE (tmppath);
 
         ret = unlink (brickpath);
 
         if ((ret < 0) && (errno != ENOENT)) {
-                gf_log ("", GF_LOG_ERROR, "Unlink failed on %s, reason: %s",
-                        brickpath, strerror(errno));
+                gf_log (this->name, GF_LOG_DEBUG, "Unlink failed on %s, "
+                        "reason: %s", brickpath, strerror(errno));
                 ret = -1;
                 goto out;
         } else {
@@ -434,15 +357,15 @@ glusterd_store_delete_brick (glusterd_volinfo_t *volinfo,
 
 out:
         if (brickinfo->shandle) {
-                glusterd_store_handle_destroy (brickinfo->shandle);
+                gf_store_handle_destroy (brickinfo->shandle);
                 brickinfo->shandle = NULL;
         }
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
 int32_t
-glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
+glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo, char *delete_path)
 {
         int32_t                 ret = 0;
         glusterd_brickinfo_t    *tmp = NULL;
@@ -451,19 +374,24 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
         DIR                     *dir = NULL;
         struct dirent           *entry = NULL;
         char                    path[PATH_MAX] = {0,};
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (volinfo);
 
         list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-                ret = glusterd_store_delete_brick (volinfo, tmp);
+                ret = glusterd_store_delete_brick (tmp, delete_path);
                 if (ret)
                         goto out;
         }
 
-        priv = THIS->private;
+        priv = this->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv);
+        snprintf (brickdir, sizeof (brickdir), "%s/%s", delete_path,
+                  GLUSTERD_BRICK_INFO_DIR);
 
         dir = opendir (brickdir);
 
@@ -474,7 +402,7 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
                           brickdir, entry->d_name);
                 ret = unlink (path);
                 if (ret && errno != ENOENT) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to unlink %s, "
+                        gf_log (this->name, GF_LOG_DEBUG, "Unable to unlink %s, "
                                 "reason: %s", path, strerror(errno));
                 }
                 glusterd_for_each_entry (entry, dir);
@@ -485,17 +413,21 @@ glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo)
         ret = rmdir (brickdir);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
-static void
+static int
 _storeslaves (dict_t *this, char *key, data_t *value, void *data)
 {
         int32_t                      ret = 0;
-        glusterd_store_handle_t  *shandle = NULL;
+        gf_store_handle_t           *shandle = NULL;
+        xlator_t                    *xl = NULL;
 
-        shandle = (glusterd_store_handle_t*)data;
+        xl = THIS;
+        GF_ASSERT (xl);
+
+        shandle = (gf_store_handle_t*)data;
 
         GF_ASSERT (shandle);
         GF_ASSERT (shandle->fd > 0);
@@ -504,32 +436,37 @@ _storeslaves (dict_t *this, char *key, data_t *value, void *data)
         GF_ASSERT (value && value->data);
 
         if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return;
+                return -1;
 
         if (!key)
-                return;
+                return -1;
         if (!value || !value->data)
-                return;
+                return -1;
 
-        gf_log ("", GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
+        gf_log (xl->name, GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
                 key, value->data);
 
-        ret = glusterd_store_save_value (shandle->fd, key, (char*)value->data);
+        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to write into store"
+                gf_log (xl->name, GF_LOG_ERROR, "Unable to write into store"
                                 " handle for path: %s", shandle->path);
-                return;
+                return -1;
         }
+        return 0;
 }
 
 
-void _storeopts (dict_t *this, char *key, data_t *value, void *data)
+int _storeopts (dict_t *this, char *key, data_t *value, void *data)
 {
         int32_t                      ret = 0;
         int32_t                      exists = 0;
-        glusterd_store_handle_t  *shandle = NULL;
+        gf_store_handle_t           *shandle = NULL;
+        xlator_t                    *xl = NULL;
+
+        xl = THIS;
+        GF_ASSERT (xl);
 
-        shandle = (glusterd_store_handle_t*)data;
+        shandle = (gf_store_handle_t*)data;
 
         GF_ASSERT (shandle);
         GF_ASSERT (shandle->fd > 0);
@@ -538,29 +475,37 @@ void _storeopts (dict_t *this, char *key, data_t *value, void *data)
         GF_ASSERT (value && value->data);
 
         if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return;
+                return -1;
 
         if (!key)
-                return;
+                return -1;
         if (!value || !value->data)
-                return;
+                return -1;
+
+        if (is_key_glusterd_hooks_friendly (key)) {
+                exists = 1;
+
+        } else {
+                exists = glusterd_check_option_exists (key, NULL);
+        }
 
-        exists = glusterd_check_option_exists (key, NULL);
         if (1 == exists) {
-                gf_log ("", GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
-                        key, value->data);
+                gf_log (xl->name, GF_LOG_DEBUG, "Storing in volinfo:key= %s, "
+                        "val=%s", key, value->data);
+
         } else {
-                gf_log ("", GF_LOG_DEBUG, "Discarding:key= %s, val=%s",
+                gf_log (xl->name, GF_LOG_DEBUG, "Discarding:key= %s, val=%s",
                         key, value->data);
-                return;
+                return 0;
         }
 
-        ret = glusterd_store_save_value (shandle->fd, key, (char*)value->data);
+        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to write into store"
-                                " handle for path: %s", shandle->path);
-                return;
+                gf_log (xl->name, GF_LOG_ERROR, "Unable to write into store"
+                        " handle for path: %s", shandle->path);
+                return -1;
         }
+        return 0;
 }
 
 int32_t
@@ -575,86 +520,89 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
         int32_t                 ret           = -1;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->type);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->brick_count);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_COUNT, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_COUNT, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->status);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STATUS, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STATUS, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->sub_count);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_SUB_COUNT,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_SUB_COUNT, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->stripe_count);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->replica_count);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
+                                   buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->version);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_VERSION,
-                                         buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_VERSION, buf);
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", volinfo->transport_type);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
-                                         buf);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_ID,
-                                         uuid_utoa (volinfo->volume_id));
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TRANSPORT, buf);
         if (ret)
                 goto out;
 
-        if (volinfo->defrag_cmd == GF_DEFRAG_CMD_STATUS)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->defrag_cmd);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DEFRAG,
-                                        buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_ID,
+                                   uuid_utoa (volinfo->volume_id));
         if (ret)
                 goto out;
 
         str = glusterd_auth_get_username (volinfo);
         if (str) {
-                ret = glusterd_store_save_value (fd,
-                                                 GLUSTERD_STORE_KEY_USERNAME,
-                                                 str);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_USERNAME,
+                                           str);
                 if (ret)
                 goto out;
         }
 
         str = glusterd_auth_get_password (volinfo);
         if (str) {
-                ret = glusterd_store_save_value (fd,
-                                                 GLUSTERD_STORE_KEY_PASSWORD,
-                                                 str);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PASSWORD,
+                                           str);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->op_version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_OP_VERSION, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->client_op_version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                                   buf);
+        if (ret)
+                goto out;
+        if (volinfo->caps) {
+                snprintf (buf, sizeof (buf), "%d", volinfo->caps);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CAPS,
+                                           buf);
                 if (ret)
                         goto out;
         }
 
 out:
         if (ret)
-                gf_log ("", GF_LOG_ERROR, "Unable to write volume values"
-                        " for %s", volinfo->volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to write volume "
+                        "values for %s", volinfo->volname);
         return ret;
 }
 
@@ -682,8 +630,8 @@ glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)
 
         glusterd_store_voldirpath_set (volinfo, voldirpath,
                                        sizeof (voldirpath));
-        ret = glusterd_store_mkdir (voldirpath);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        ret = gf_store_mkdir (voldirpath);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
@@ -691,7 +639,7 @@ int32_t
 glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
 {
         int32_t                         ret = -1;
-        glusterd_store_handle_t         *shandle = NULL;
+        gf_store_handle_t              *shandle = NULL;
         GF_ASSERT (fd > 0);
         GF_ASSERT (volinfo);
         GF_ASSERT (volinfo->shandle);
@@ -707,7 +655,7 @@ glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
         dict_foreach (volinfo->gsync_slaves, _storeslaves, shandle);
         shandle->fd = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -718,7 +666,7 @@ glusterd_store_rbstatepath_set (glusterd_volinfo_t *volinfo, char *rbstatepath,
         char    voldirpath[PATH_MAX] = {0,};
         GF_ASSERT (volinfo);
         GF_ASSERT (rbstatepath);
-        GF_ASSERT (len >= PATH_MAX);
+        GF_ASSERT (len <= PATH_MAX);
 
         glusterd_store_voldirpath_set (volinfo, voldirpath,
                                        sizeof (voldirpath));
@@ -733,13 +681,43 @@ glusterd_store_volfpath_set (glusterd_volinfo_t *volinfo, char *volfpath,
         char    voldirpath[PATH_MAX] = {0,};
         GF_ASSERT (volinfo);
         GF_ASSERT (volfpath);
-        GF_ASSERT (len >= PATH_MAX);
+        GF_ASSERT (len <= PATH_MAX);
 
         glusterd_store_voldirpath_set (volinfo, voldirpath,
                                        sizeof (voldirpath));
         snprintf (volfpath, len, "%s/%s", voldirpath, GLUSTERD_VOLUME_INFO_FILE);
 }
 
+static void
+glusterd_store_node_state_path_set (glusterd_volinfo_t *volinfo,
+                                    char *node_statepath, size_t len)
+{
+        char    voldirpath[PATH_MAX] = {0,};
+        GF_ASSERT (volinfo);
+        GF_ASSERT (node_statepath);
+        GF_ASSERT (len <= PATH_MAX);
+
+        glusterd_store_voldirpath_set (volinfo, voldirpath,
+                                       sizeof (voldirpath));
+        snprintf (node_statepath, len, "%s/%s", voldirpath,
+                  GLUSTERD_NODE_STATE_FILE);
+}
+
+static void
+glusterd_store_quota_conf_path_set (glusterd_volinfo_t *volinfo,
+                                    char *quota_conf_path, size_t len)
+{
+        char    voldirpath[PATH_MAX] = {0,};
+        GF_ASSERT (volinfo);
+        GF_ASSERT (quota_conf_path);
+        GF_ASSERT (len <= PATH_MAX);
+
+        glusterd_store_voldirpath_set (volinfo, voldirpath,
+                                       sizeof (voldirpath));
+        snprintf (quota_conf_path, len, "%s/%s", voldirpath,
+                  GLUSTERD_VOLUME_QUOTA_CONFIG);
+}
+
 int32_t
 glusterd_store_create_rbstate_shandle_on_absence (glusterd_volinfo_t *volinfo)
 {
@@ -749,8 +727,8 @@ glusterd_store_create_rbstate_shandle_on_absence (glusterd_volinfo_t *volinfo)
         GF_ASSERT (volinfo);
 
         glusterd_store_rbstatepath_set (volinfo, rbstatepath, sizeof (rbstatepath));
-        ret = glusterd_store_handle_create_on_absence (&volinfo->rb_shandle,
-                                                       rbstatepath);
+        ret = gf_store_handle_create_on_absence (&volinfo->rb_shandle,
+                                                 rbstatepath);
         return ret;
 }
 
@@ -763,12 +741,44 @@ glusterd_store_create_vol_shandle_on_absence (glusterd_volinfo_t *volinfo)
         GF_ASSERT (volinfo);
 
         glusterd_store_volfpath_set (volinfo, volfpath, sizeof (volfpath));
-        ret = glusterd_store_handle_create_on_absence (&volinfo->shandle,
-                                                       volfpath);
+        ret = gf_store_handle_create_on_absence (&volinfo->shandle, volfpath);
+        return ret;
+}
+
+int32_t
+glusterd_store_create_nodestate_sh_on_absence (glusterd_volinfo_t *volinfo)
+{
+        char            node_state_path[PATH_MAX] = {0};
+        int32_t         ret                   = 0;
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_node_state_path_set (volinfo, node_state_path,
+                                            sizeof (node_state_path));
+        ret =
+          gf_store_handle_create_on_absence (&volinfo->node_state_shandle,
+                                             node_state_path);
+
         return ret;
 }
 
 int32_t
+glusterd_store_create_quota_conf_sh_on_absence (glusterd_volinfo_t *volinfo)
+{
+        char            quota_conf_path[PATH_MAX] = {0};
+        int32_t         ret                       = 0;
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_quota_conf_path_set (volinfo, quota_conf_path,
+                                            sizeof (quota_conf_path));
+        ret =
+          gf_store_handle_create_on_absence (&volinfo->quota_conf_shandle,
+                                             quota_conf_path);
+
+        return ret;
+}
+int32_t
 glusterd_store_brickinfos (glusterd_volinfo_t *volinfo, int vol_fd)
 {
         int32_t                 ret = 0;
@@ -785,7 +795,7 @@ glusterd_store_brickinfos (glusterd_volinfo_t *volinfo, int vol_fd)
                 brick_count++;
         }
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -793,36 +803,56 @@ int32_t
 glusterd_store_rbstate_write (int fd, glusterd_volinfo_t *volinfo)
 {
         int     ret             = -1;
+        int     port            = 0;
         char    buf[PATH_MAX]   = {0, };
 
         GF_ASSERT (fd > 0);
         GF_ASSERT (volinfo);
 
-        snprintf (buf, sizeof (buf), "%d", volinfo->rb_status);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_STATUS,
-                                         buf);
+        snprintf (buf, sizeof (buf), "%d", volinfo->rep_brick.rb_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_STATUS, buf);
         if (ret)
                 goto out;
 
-        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+        if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
 
                 snprintf (buf, sizeof (buf), "%s:%s",
-                          volinfo->src_brick->hostname,
-                          volinfo->src_brick->path);
-                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                                                 buf);
+                          volinfo->rep_brick.src_brick->hostname,
+                          volinfo->rep_brick.src_brick->path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                                           buf);
                 if (ret)
                         goto out;
 
                 snprintf (buf, sizeof (buf), "%s:%s",
-                          volinfo->dst_brick->hostname,
-                          volinfo->dst_brick->path);
-                ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                                                 buf);
+                          volinfo->rep_brick.dst_brick->hostname,
+                          volinfo->rep_brick.dst_brick->path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                                           buf);
                 if (ret)
                         goto out;
+
+                switch (volinfo->transport_type) {
+                        case GF_TRANSPORT_RDMA:
+                                port = volinfo->rep_brick.dst_brick->rdma_port;
+                                break;
+
+                        case GF_TRANSPORT_TCP:
+                        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                                port = volinfo->rep_brick.dst_brick->port;
+                                break;
+                }
+
+                snprintf (buf, sizeof (buf), "%d", port);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_PORT,
+                                           buf);
+                if (ret)
+                        goto out;
+                uuid_unparse (volinfo->rep_brick.rb_id, buf);
+                ret = gf_store_save_value (fd, GF_REPLACE_BRICK_TID_KEY, buf);
         }
 
+        ret = 0;
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -835,7 +865,7 @@ glusterd_store_perform_rbstate_store (glusterd_volinfo_t *volinfo)
         int32_t                     ret = -1;
         GF_ASSERT (volinfo);
 
-        fd = glusterd_store_mkstemp (volinfo->rb_shandle);
+        fd = gf_store_mkstemp (volinfo->rb_shandle);
         if (fd <= 0) {
                 ret = -1;
                 goto out;
@@ -845,14 +875,97 @@ glusterd_store_perform_rbstate_store (glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_store_rename_tmppath (volinfo->rb_shandle);
+        ret = gf_store_rename_tmppath (volinfo->rb_shandle);
+        if (ret)
+                goto out;
 
 out:
         if (ret && (fd > 0))
-                glusterd_store_unlink_tmppath (volinfo->rb_shandle);
+                gf_store_unlink_tmppath (volinfo->rb_shandle);
         if (fd > 0)
                 close (fd);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+_gd_store_rebalance_dict (dict_t *dict, char *key, data_t *value, void *data)
+{
+        int             ret = -1;
+        int             fd = 0;
+
+        fd = *(int *)data;
+
+        ret = gf_store_save_value (fd, key, value->data);
+
+        return ret;
+}
+
+int32_t
+glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
+{
+        int     ret             = -1;
+        char    buf[PATH_MAX]   = {0, };
+
+        GF_ASSERT (fd > 0);
+        GF_ASSERT (volinfo);
+
+        if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_STATUS) {
+                ret = 0;
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.defrag_cmd);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DEFRAG, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.op);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_DEFRAG_OP, buf);
+        if (ret)
+                goto out;
+
+        uuid_unparse (volinfo->rebal.rebalance_id, buf);
+        ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
+        if (ret)
+                goto out;
+
+        if (volinfo->rebal.dict) {
+                dict_foreach (volinfo->rebal.dict, _gd_store_rebalance_dict,
+                              &fd);
+        }
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo)
+{
+        int                         fd = -1;
+        int32_t                     ret = -1;
+        GF_ASSERT (volinfo);
+
+        fd = gf_store_mkstemp (volinfo->node_state_shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_node_state_write (fd, volinfo);
+        if (ret)
+                goto out;
+
+        ret = gf_store_rename_tmppath (volinfo->node_state_shandle);
+        if (ret)
+                goto out;
+
+out:
+        if (ret && (fd > 0))
+                gf_store_unlink_tmppath (volinfo->node_state_shandle);
+        if (fd > 0)
+                close (fd);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -863,7 +976,7 @@ glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo)
         int32_t                     ret = -1;
         GF_ASSERT (volinfo);
 
-        fd = glusterd_store_mkstemp (volinfo->shandle);
+        fd = gf_store_mkstemp (volinfo->shandle);
         if (fd <= 0) {
                 ret = -1;
                 goto out;
@@ -877,13 +990,12 @@ glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_store_rename_tmppath (volinfo->shandle);
 out:
         if (ret && (fd > 0))
-                glusterd_store_unlink_tmppath (volinfo->shandle);
+                gf_store_unlink_tmppath (volinfo->shandle);
         if (fd > 0)
                 close (fd);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -899,7 +1011,86 @@ glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
         case GLUSTERD_VOLINFO_VER_AC_INCREMENT:
                 volinfo->version++;
         break;
+        case GLUSTERD_VOLINFO_VER_AC_DECREMENT:
+                volinfo->version--;
+        break;
+        }
+}
+
+void
+glusterd_store_bricks_cleanup_tmp (glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t    *brickinfo           = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                gf_store_unlink_tmppath (brickinfo->shandle);
+        }
+}
+
+void
+glusterd_store_volume_cleanup_tmp (glusterd_volinfo_t *volinfo)
+{
+        GF_ASSERT (volinfo);
+
+        glusterd_store_bricks_cleanup_tmp (volinfo);
+
+        gf_store_unlink_tmppath (volinfo->shandle);
+
+        gf_store_unlink_tmppath (volinfo->rb_shandle);
+
+        gf_store_unlink_tmppath (volinfo->node_state_shandle);
+}
+
+int32_t
+glusterd_store_brickinfos_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int                      ret            = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = gf_store_rename_tmppath (brickinfo->shandle);
+                if (ret)
+                        goto out;
         }
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_volinfo_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int ret = -1;
+        GF_ASSERT (volinfo);
+
+        ret = gf_store_rename_tmppath (volinfo->shandle);
+        if (ret)
+                goto out;
+
+out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't rename "
+                        "temporary file(s): Reason %s", strerror (errno));
+        return ret;
+}
+
+int32_t
+glusterd_store_volume_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int ret = -1;
+        GF_ASSERT (volinfo);
+
+        ret = glusterd_store_brickinfos_atomic_update (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_volinfo_atomic_update (volinfo);
+
+out:
+        return ret;
 }
 
 int32_t
@@ -922,20 +1113,39 @@ glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t a
         if (ret)
                 goto out;
 
+        ret = glusterd_store_create_nodestate_sh_on_absence (volinfo);
+        if (ret)
+                goto out;
+
         ret = glusterd_store_perform_volume_store (volinfo);
         if (ret)
                 goto out;
 
+        ret = glusterd_store_volume_atomic_update (volinfo);
+        if (ret) {
+                glusterd_perform_volinfo_version_action (volinfo,
+                                                         GLUSTERD_VOLINFO_VER_AC_DECREMENT);
+                goto out;
+        }
+
         ret = glusterd_store_perform_rbstate_store (volinfo);
         if (ret)
                 goto out;
 
+        ret = glusterd_store_perform_node_state_store (volinfo);
+        if (ret)
+                goto out;
+
         //checksum should be computed at the end
-        ret = glusterd_volume_compute_cksum (volinfo);
+        ret = glusterd_compute_cksum (volinfo, _gf_false);
         if (ret)
                 goto out;
+
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (ret)
+                glusterd_store_volume_cleanup_tmp (volinfo);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -944,39 +1154,73 @@ out:
 int32_t
 glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
 {
-        char    pathname[PATH_MAX] = {0,};
-        int32_t ret = 0;
-        glusterd_conf_t *priv = NULL;
-        DIR     *dir = NULL;
-        struct dirent *entry = NULL;
-        char path[PATH_MAX] = {0,};
-        struct stat     st = {0, };
+        char             pathname[PATH_MAX]    = {0,};
+        int32_t          ret                   = 0;
+        glusterd_conf_t *priv                  = NULL;
+        DIR             *dir                   = NULL;
+        struct dirent   *entry                 = NULL;
+        char             path[PATH_MAX]        = {0,};
+        char             delete_path[PATH_MAX] = {0,};
+        char             trashdir[PATH_MAX]    = {0,};
+        struct stat      st                    = {0, };
+        xlator_t        *this                  = NULL;
+        gf_boolean_t     rename_fail           = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (volinfo);
-        priv = THIS->private;
+        priv = this->private;
 
         GF_ASSERT (priv);
-        snprintf (pathname, sizeof (pathname), "%s/vols/%s", priv->workdir,
-                  volinfo->volname);
 
-        dir = opendir (pathname);
-        if (!dir)
+        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
+
+        snprintf (delete_path, sizeof (delete_path),
+                  "%s/"GLUSTERD_TRASH"/%s.deleted", priv->workdir,
+                  uuid_utoa (volinfo->volume_id));
+
+        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
+                  priv->workdir);
+
+        ret = mkdir (trashdir, 0777);
+        if (ret && errno != EEXIST) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
+                        "directory, reason : %s", strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = rename (pathname, delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to rename volume "
+                        "directory for volume %s", volinfo->volname);
+                rename_fail = _gf_true;
                 goto out;
-        ret = glusterd_store_remove_bricks (volinfo);
+        }
+
+        dir = opendir (delete_path);
+        if (!dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to open directory %s."
+                        " Reason : %s", delete_path, strerror (errno));
+                ret = 0;
+                goto out;
+        }
+        ret = glusterd_store_remove_bricks (volinfo, delete_path);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Remove bricks failed for %s",
+                gf_log (this->name, GF_LOG_DEBUG, "Remove bricks failed for %s",
                         volinfo->volname);
         }
 
         glusterd_for_each_entry (entry, dir);
         while (entry) {
 
-                snprintf (path, PATH_MAX, "%s/%s", pathname, entry->d_name);
+                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
                 ret = stat (path, &st);
                 if (ret == -1) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to stat entry: %s:%s",
-                                path, strerror (errno));
+                        gf_log (this->name, GF_LOG_DEBUG, "Failed to stat "
+                                "entry %s : %s", path, strerror (errno));
                         goto stat_failed;
                 }
 
@@ -985,12 +1229,13 @@ glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
                 else
                         ret = unlink (path);
 
-                if (ret)
-                        gf_log ("", GF_LOG_INFO, "errno:%d (%s)", errno,
-                                strerror (errno));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, " Failed to remove "
+                                "%s. Reason : %s", path, strerror (errno));
+                }
 
-                gf_log ("", GF_LOG_INFO, "%s %s",
-                                ret?"Failed to remove":"Removed",
+                gf_log (this->name, GF_LOG_DEBUG, "%s %s",
+                                ret ? "Failed to remove":"Removed",
                                 entry->d_name);
 stat_failed:
                 memset (path, 0, sizeof(path));
@@ -999,317 +1244,216 @@ stat_failed:
 
         ret = closedir (dir);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Failed to close dir, errno:%d",
-                        errno);
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close dir %s. "
+                        "Reason : %s",delete_path, strerror (errno));
         }
 
-        ret = rmdir (pathname);
+        ret = rmdir (delete_path);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Failed to rmdir: %s, err: %s",
-                        pathname, strerror (errno));
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s,err: %s",
+                        delete_path, strerror (errno));
+        }
+        ret = rmdir (trashdir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s, Reason:"
+                        " %s", trashdir, strerror (errno));
         }
-
 
 out:
         if (volinfo->shandle) {
-                glusterd_store_handle_destroy (volinfo->shandle);
+                gf_store_handle_destroy (volinfo->shandle);
                 volinfo->shandle = NULL;
         }
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        ret = (rename_fail == _gf_true) ? -1: 0;
 
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 
 int
-glusterd_store_read_and_tokenize (FILE *file, char *str,
-                                  char **iter_key, char **iter_val,
-                                  glusterd_store_op_errno_t *store_errno)
+glusterd_store_global_info (xlator_t *this)
 {
-        int32_t  ret = -1;
+        int                     ret                     = -1;
+        glusterd_conf_t         *conf                   = NULL;
+        char                    op_version_str[15]      = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        gf_store_handle_t       *handle                 = NULL;
+        char                    *uuid_str               = NULL;
 
-        GF_ASSERT (file);
-        GF_ASSERT (str);
-        GF_ASSERT (iter_key);
-        GF_ASSERT (iter_val);
-        GF_ASSERT (store_errno);
+        conf = this->private;
 
-        ret = fscanf (file, "%s", str);
-        if (ret <= 0 || feof (file)) {
-                ret = -1;
-                *store_errno = GD_STORE_EOF;
-                goto out;
-        }
-
-        *iter_key = strtok (str, "=");
-        if (*iter_key == NULL) {
-                ret = -1;
-                *store_errno = GD_STORE_KEY_NULL;
+        uuid_str = gf_strdup (uuid_utoa (MY_UUID));
+        if (!uuid_str)
                 goto out;
-        }
 
-        *iter_val = strtok (NULL, "=");
-        if (*iter_key == NULL) {
-                ret = -1;
-                *store_errno = GD_STORE_VALUE_NULL;
-                goto out;
-        }
-
-        *store_errno = GD_STORE_SUCCESS;
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-glusterd_store_retrieve_value (glusterd_store_handle_t *handle,
-                               char *key, char **value)
-{
-        int32_t         ret = -1;
-        char            *scan_str = NULL;
-        char            *iter_key = NULL;
-        char            *iter_val = NULL;
-        char            *free_str = NULL;
-        struct stat     st        = {0,};
-        glusterd_store_op_errno_t store_errno = GD_STORE_SUCCESS;
-
-        GF_ASSERT (handle);
-
-        handle->fd = open (handle->path, O_RDWR);
+        if (!conf->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", conf->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_new (path, &handle);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get store handle");
+                        goto out;
+                }
 
-        if (!handle->read)
-                handle->read = fdopen (handle->fd, "r");
+                conf->handle = handle;
+        } else
+                handle = conf->handle;
 
-        if (!handle->read) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %d",
-                        handle->path, errno);
+        /* These options need to be available for all users */
+        ret = chmod (handle->path, 0644);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "chmod error for %s: %s",
+                        GLUSTERD_INFO_FILE, strerror (errno));
                 goto out;
         }
 
-        ret = fstat (handle->fd, &st);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_WARNING,
-                        "stat on file failed");
+        handle->fd = gf_store_mkstemp (handle);
+        if (handle->fd <= 0) {
                 ret = -1;
-                store_errno = GD_STORE_STAT_FAILED;
                 goto out;
         }
 
-        scan_str = GF_CALLOC (1, st.st_size,
-                              gf_gld_mt_char);
-        if (scan_str == NULL) {
-                ret = -1;
-                store_errno = GD_STORE_ENOMEM;
+        ret = gf_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY,
+                                   uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Storing uuid failed ret = %d", ret);
                 goto out;
         }
 
-        free_str = scan_str;
-
-        do {
-                ret = glusterd_store_read_and_tokenize (handle->read, scan_str,
-                                                        &iter_key, &iter_val,
-                                                        &store_errno);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                gf_log ("", GF_LOG_DEBUG, "key %s read", iter_key);
-
-                if (!strcmp (key, iter_key)) {
-                        gf_log ("", GF_LOG_DEBUG, "key %s found", key);
-                        ret = 0;
-                        if (iter_val)
-                                *value = gf_strdup (iter_val);
-                        goto out;
-                }
-        } while (1);
-out:
-        if (handle->fd > 0) {
-                close (handle->fd);
-                handle->read = NULL;
-        }
-
-        if (free_str)
-                GF_FREE (free_str);
-
-        return ret;
-}
-
-int32_t
-glusterd_store_save_value (int fd, char *key, char *value)
-{
-        int32_t         ret = -1;
-        FILE           *fp  = NULL;
-
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (key);
-        GF_ASSERT (value);
-
-        fp = fdopen (fd, "a+");
-        if (fp == NULL) {
-                gf_log ("", GF_LOG_WARNING, "fdopen failed.");
-                ret = -1;
+        snprintf (op_version_str, 15, "%d", conf->op_version);
+        ret = gf_store_save_value (handle->fd, GD_OP_VERSION_KEY,
+                                   op_version_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing op-version failed ret = %d", ret);
                 goto out;
         }
 
-        ret = fprintf (fp, "%s=%s\n", key, value);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "Unable to store key: %s,"
-                        "value: %s, error: %s", key, value,
-                        strerror (errno));
-                ret = -1;
-                goto out;
-        }
+        ret = gf_store_rename_tmppath (handle);
+out:
+        if (handle) {
+                if (ret && (handle->fd > 0))
+                        gf_store_unlink_tmppath (handle);
 
-        ret = fflush (fp);
-        if (feof (fp)) {
-                gf_log ("", GF_LOG_WARNING,
-                        "fflush failed, error: %s",
-                        strerror (errno));
-                ret = -1;
-                goto out;
+                if (handle->fd > 0) {
+                        close (handle->fd);
+                        handle->fd = 0;
+                }
         }
 
-        ret = 0;
-out:
+        if (uuid_str)
+                GF_FREE (uuid_str);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to store glusterd global-info");
 
-        gf_log ("", GF_LOG_DEBUG, "returning: %d", ret);
         return ret;
 }
 
-int32_t
-glusterd_store_handle_new (char *path, glusterd_store_handle_t **handle)
+int
+glusterd_retrieve_op_version (xlator_t *this, int *op_version)
 {
-        int32_t                 ret = -1;
-        glusterd_store_handle_t *shandle = NULL;
-        int                     fd = -1;
-        char                    *spath = NULL;
+        char                    *op_version_str = NULL;
+        glusterd_conf_t         *priv           = NULL;
+        int                     ret             = -1;
+        int                     tmp_version     = 0;
+        char                    *tmp            = NULL;
+        char                    path[PATH_MAX]  = {0,};
+        gf_store_handle_t       *handle         = NULL;
 
-        shandle = GF_CALLOC (1, sizeof (*shandle), gf_gld_mt_store_handle_t);
-        if (!shandle)
-                goto out;
-
-        spath = gf_strdup (path);
-
-        if (!spath)
-                goto out;
-
-        fd = open (path, O_RDWR | O_CREAT | O_APPEND, 0600);
-        if (fd <= 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to open file: %s, "
-                        "error: %s", path, strerror (errno));
-                goto out;
-        }
-
-        shandle->path = spath;
-        *handle = shandle;
-
-        ret = 0;
+        priv = this->private;
 
-out:
-        if (fd > 0)
-                close (fd);
+        if (!priv->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_retrieve (path, &handle);
 
-        if (ret == -1) {
-                if (spath)
-                        GF_FREE (spath);
-                if (shandle) {
-                        GF_FREE (shandle);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store "
+                                "handle!");
+                        goto out;
                 }
-        }
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
 
-int
-glusterd_store_handle_retrieve (char *path, glusterd_store_handle_t **handle)
-{
-        int32_t                 ret = -1;
-        struct stat statbuf = {0};
+                priv->handle = handle;
+        }
 
-        ret = stat (path, &statbuf);
+        ret = gf_store_retrieve_value (priv->handle, GD_OP_VERSION_KEY,
+                                       &op_version_str);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to retrieve store "
-                        "handle for %s, error: %s", path, strerror (errno));
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No previous op_version present");
                 goto out;
         }
-        ret =  glusterd_store_handle_new (path, handle);
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_store_handle_destroy (glusterd_store_handle_t *handle)
-{
-        int32_t                 ret = -1;
 
-        if (!handle) {
-                ret = 0;
+        tmp_version = strtol (op_version_str, &tmp, 10);
+        if ((tmp_version <= 0) || (tmp && strlen (tmp) > 1)) {
+                gf_log (this->name, GF_LOG_WARNING, "invalid version number");
                 goto out;
         }
 
-        GF_FREE (handle->path);
-
-        GF_FREE (handle);
+        *op_version = tmp_version;
 
         ret = 0;
-
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (op_version_str)
+                GF_FREE (op_version_str);
 
         return ret;
 }
 
-int32_t
-glusterd_store_uuid ()
+static int
+glusterd_restore_op_version (xlator_t *this)
 {
-        glusterd_conf_t *priv = NULL;
-        char            path[PATH_MAX] = {0,};
-        int32_t         ret = -1;
-        glusterd_store_handle_t *handle = NULL;
-
-        priv = THIS->private;
-
-        snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                  GLUSTERD_INFO_FILE);
+        glusterd_conf_t *conf = NULL;
+        int ret = 0;
+        int op_version = 0;
 
-        if (!priv->handle) {
-                ret = glusterd_store_handle_new (path, &handle);
+        conf = this->private;
 
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get store"
-                                " handle!");
+        ret = glusterd_retrieve_op_version (this, &op_version);
+        if (!ret) {
+                if ((op_version < GD_OP_VERSION_MIN) ||
+                    (op_version > GD_OP_VERSION_MAX)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "wrong op-version (%d) retrieved", op_version);
+                        ret = -1;
                         goto out;
                 }
-
-                priv->handle = handle;
-        } else {
-                handle = priv->handle;
-        }
-
-        handle->fd = open (handle->path, O_RDWR | O_CREAT | O_TRUNC, 0600);
-        if (handle->fd <= 0) {
-                ret = -1;
-                goto out;
-        }
-        ret = glusterd_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY,
-                                         uuid_utoa (priv->uuid));
-
+                conf->op_version = op_version;
+                gf_log ("glusterd", GF_LOG_INFO,
+                        "retrieved op-version: %d", conf->op_version);
+                goto out;
+        }
+
+        /* op-version can be missing from the store file in 2 cases,
+         * 1. This is a new install of glusterfs
+         * 2. This is an upgrade of glusterfs from a version without op-version
+         *    to a version with op-version (eg. 3.3 -> 3.4)
+         *
+         * Detection of a new install or an upgrade from an older install can be
+         * done by checking for the presence of the its peer-id in the store
+         * file.  If peer-id is present, the installation is an upgrade else, it
+         * is a new install.
+         *
+         * For case 1, set op-version to GD_OP_VERSION_MAX.
+         * For case 2, set op-version to GD_OP_VERSION_MIN.
+         */
+        ret = glusterd_retrieve_uuid();
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Storing uuid failed"
-                        "ret = %d", ret);
-                goto out;
+                gf_log (this->name, GF_LOG_INFO, "Detected new install. Setting"
+                        " op-version to maximum : %d", GD_OP_VERSION_MAX);
+                conf->op_version = GD_OP_VERSION_MAX;
+        } else {
+                gf_log (this->name, GF_LOG_INFO, "Upgrade detected. Setting"
+                        " op-version to minimum : %d", GD_OP_VERSION_MIN);
+                conf->op_version = GD_OP_VERSION_MIN;
         }
-
-
+        ret = 0;
 out:
-        if (handle->fd > 0) {
-                close (handle->fd);
-                handle->fd = 0;
-        }
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -1318,20 +1462,19 @@ glusterd_retrieve_uuid ()
 {
         char            *uuid_str = NULL;
         int32_t         ret = -1;
-        glusterd_store_handle_t *handle = NULL;
+        gf_store_handle_t *handle = NULL;
         glusterd_conf_t *priv = NULL;
         char            path[PATH_MAX] = {0,};
 
         priv = THIS->private;
 
-
         if (!priv->handle) {
                 snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
                           GLUSTERD_INFO_FILE);
-                ret = glusterd_store_handle_retrieve (path, &handle);
+                ret = gf_store_handle_retrieve (path, &handle);
 
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get store "
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store"
                                 "handle!");
                         goto out;
                 }
@@ -1339,262 +1482,29 @@ glusterd_retrieve_uuid ()
                 priv->handle = handle;
         }
 
-        ret = glusterd_store_retrieve_value (priv->handle,
-                                             GLUSTERD_STORE_UUID_KEY,
-                                             &uuid_str);
+        ret = gf_store_retrieve_value (priv->handle, GLUSTERD_STORE_UUID_KEY,
+                                       &uuid_str);
 
         if (ret) {
-                gf_log ("", GF_LOG_INFO, "No previous uuid is present");
+                gf_log ("", GF_LOG_DEBUG, "No previous uuid is present");
                 goto out;
         }
 
         uuid_parse (uuid_str, priv->uuid);
 
 out:
-        if (uuid_str)
-                GF_FREE (uuid_str);
+        GF_FREE (uuid_str);
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd_store_iter_new (glusterd_store_handle_t  *shandle,
-                         glusterd_store_iter_t  **iter)
-{
-        int32_t                 ret = -1;
-        glusterd_store_iter_t   *tmp_iter = NULL;
-        int                     fd = -1;
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (iter);
-
-        tmp_iter = GF_CALLOC (1, sizeof (*tmp_iter),
-                             gf_gld_mt_store_iter_t);
-
-        if (!tmp_iter) {
-                gf_log ("", GF_LOG_ERROR, "Out of Memory");
-                goto out;
-        }
-
-        fd = open (shandle->path, O_RDWR);
-
-        if (fd < 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open %s, errno: %d",
-                        shandle->path, errno);
-                goto out;
-        }
-
-        tmp_iter->fd = fd;
-
-        tmp_iter->file = fdopen (tmp_iter->fd, "r");
-
-        if (!tmp_iter->file) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open file %s errno: %d",
-                        shandle->path, errno);
-                goto out;
-        }
-
-        strncpy (tmp_iter->filepath, shandle->path, sizeof (tmp_iter->filepath));
-        *iter = tmp_iter;
-        ret = 0;
-
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_store_validate_key_value (char *storepath, char *key, char*val,
-                                   glusterd_store_op_errno_t *op_errno)
-{
-        int ret = 0;
-
-        GF_ASSERT (op_errno);
-        GF_ASSERT (storepath);
-
-        if ((key == NULL) && (val == NULL)) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Glusterd store may be "
-                        "corrupted, Invalid key and value (null) in %s",
-                        storepath);
-                *op_errno = GD_STORE_KEY_VALUE_NULL;
-        } else if (key == NULL) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Glusterd store may be "
-                        "corrupted, Invalid key (null) in %s", storepath);
-                *op_errno = GD_STORE_KEY_NULL;
-        } else if (val == NULL) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, "Glusterd store may be "
-                        "corrupted, Invalid value (null) for key %s in %s",
-                        key, storepath);
-                *op_errno = GD_STORE_VALUE_NULL;
-        } else {
-                ret = 0;
-                *op_errno = GD_STORE_SUCCESS;
-        }
-
-        return ret;
-}
-
-int32_t
-glusterd_store_iter_get_next (glusterd_store_iter_t *iter,
-                              char  **key, char **value,
-                              glusterd_store_op_errno_t *op_errno)
-{
-        int32_t         ret = -1;
-        char            *scan_str = NULL;
-        char            *free_str = NULL;
-        char            *iter_key = NULL;
-        char            *iter_val = NULL;
-        struct stat     st        = {0,};
-        glusterd_store_op_errno_t store_errno = GD_STORE_SUCCESS;
-
-        GF_ASSERT (iter);
-        GF_ASSERT (iter->file);
-        GF_ASSERT (key);
-        GF_ASSERT (value);
-
-        ret = fstat (iter->fd, &st);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_WARNING,
-                        "stat on file failed");
-                ret = -1;
-                store_errno = GD_STORE_STAT_FAILED;
-                goto out;
-        }
-
-        scan_str = GF_CALLOC (1, st.st_size,
-                              gf_gld_mt_char);
-        if (scan_str == NULL) {
-                ret = -1;
-                store_errno = GD_STORE_ENOMEM;
-                goto out;
-        }
-
-        *key = NULL;
-        *value = NULL;
-
-        free_str = scan_str;
-
-        ret = glusterd_store_read_and_tokenize (iter->file, scan_str,
-                                                &iter_key, &iter_val,
-                                                &store_errno);
-        if (ret < 0) {
-                goto out;
-        }
-
-
-        ret = glusterd_store_validate_key_value (iter->filepath, iter_key,
-                                                 iter_val, &store_errno);
-        if (ret)
-                goto out;
-
-        *value = gf_strdup (iter_val);
-
-        *key   = gf_strdup (iter_key);
-        if (!iter_key || !iter_val) {
-                ret = -1;
-                store_errno = GD_STORE_ENOMEM;
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        if (ret) {
-                if (*key) {
-                        GF_FREE (*key);
-                        *key = NULL;
-                }
-                if (*value) {
-                        GF_FREE (*value);
-                        *value = NULL;
-                }
-        }
-        if (free_str)
-                GF_FREE (free_str);
-        if (op_errno)
-                *op_errno = store_errno;
-
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_store_iter_get_matching (glusterd_store_iter_t *iter,
-                              char  *key, char **value)
-{
-        int32_t ret = -1;
-        char    *tmp_key = NULL;
-        char    *tmp_value = NULL;
-
-        ret = glusterd_store_iter_get_next (iter, &tmp_key, &tmp_value,
-                                            NULL);
-        while (!ret) {
-                if (!strncmp (key, tmp_key, strlen (key))){
-                        *value = tmp_value;
-                        GF_FREE (tmp_key);
-                        goto out;
-                }
-                GF_FREE (tmp_key);
-                GF_FREE (tmp_value);
-                ret = glusterd_store_iter_get_next (iter, &tmp_key,
-                                                    &tmp_value, NULL);
-        }
-out:
-        return ret;
-}
-
-int32_t
-glusterd_store_iter_destroy (glusterd_store_iter_t *iter)
-{
-        int32_t         ret = -1;
-
-        GF_ASSERT (iter);
-        GF_ASSERT (iter->fd > 0);
-
-        ret = fclose (iter->file);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to close fd: %d, ret: %d, "
-                        "errno: %d" ,iter->fd, ret, errno);
-        }
-
-        GF_FREE (iter);
-
-        return ret;
-}
-
-char*
-glusterd_store_strerror (glusterd_store_op_errno_t op_errno)
-{
-        switch (op_errno) {
-        case GD_STORE_SUCCESS:
-                return "Success";
-        case GD_STORE_KEY_NULL:
-                return "Invalid Key";
-        case GD_STORE_VALUE_NULL:
-                return "Invalid Value";
-        case GD_STORE_KEY_VALUE_NULL:
-                return "Invalid Key and Value";
-        case GD_STORE_EOF:
-                return "No data";
-        case GD_STORE_ENOMEM:
-                return "No memory";
-        default:
-                return "Invalid errno";
-        }
-        return "Invalid errno";
-}
-
-int32_t
 glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
 {
 
         int32_t                 ret = 0;
         glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_store_iter_t   *iter = NULL;
+        gf_store_iter_t         *iter = NULL;
         char                    *key = NULL;
         char                    *value = NULL;
         char                    brickdir[PATH_MAX] = {0,};
@@ -1602,10 +1512,11 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
         glusterd_conf_t         *priv = NULL;
         int32_t                 brick_count = 0;
         char                    tmpkey[4096] = {0,};
-        glusterd_store_iter_t   *tmpiter = NULL;
+        gf_store_iter_t         *tmpiter = NULL;
         char                    *tmpvalue = NULL;
-        struct pmap_registry *pmap = NULL;
-        glusterd_store_op_errno_t       op_errno = GD_STORE_SUCCESS;
+        struct pmap_registry    *pmap = NULL;
+        int                      brickid = 0;
+        gf_store_op_errno_t     op_errno = GD_STORE_SUCCESS;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (volinfo->volname);
@@ -1614,7 +1525,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
 
         GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv)
 
-        ret = glusterd_store_iter_new (volinfo->shandle, &tmpiter);
+        ret = gf_store_iter_new (volinfo->shandle, &tmpiter);
 
         if (ret)
                 goto out;
@@ -1626,30 +1537,28 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         goto out;
                 snprintf (tmpkey, sizeof (tmpkey), "%s-%d",
                           GLUSTERD_STORE_KEY_VOL_BRICK,brick_count);
-                ret = glusterd_store_iter_get_matching (tmpiter, tmpkey,
-                                                        &tmpvalue);
+                ret = gf_store_iter_get_matching (tmpiter, tmpkey, &tmpvalue);
                 snprintf (path, sizeof (path), "%s/%s", brickdir, tmpvalue);
 
                 GF_FREE (tmpvalue);
 
                 tmpvalue = NULL;
 
-                ret = glusterd_store_handle_retrieve (path, &brickinfo->shandle);
+                ret = gf_store_handle_retrieve (path, &brickinfo->shandle);
 
                 if (ret)
                         goto out;
 
-                ret = glusterd_store_iter_new (brickinfo->shandle, &iter);
+                ret = gf_store_iter_new (brickinfo->shandle, &iter);
 
                 if (ret)
                         goto out;
 
-                ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                    &op_errno);
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
                 if (ret) {
                         gf_log ("glusterd", GF_LOG_ERROR, "Unable to iterate "
                                 "the store for brick: %s, reason: %s", path,
-                                glusterd_store_strerror (op_errno));
+                                gf_store_strerror (op_errno));
                         goto out;
                 }
                 while (!ret) {
@@ -1663,22 +1572,48 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PORT,
                                     strlen (GLUSTERD_STORE_KEY_BRICK_PORT))) {
                                 gf_string2int (value, &brickinfo->port);
-                                /* This is required to have proper ports
-                                   assigned to bricks after restart */
-                                pmap = pmap_registry_get (THIS);
-                                if (pmap->last_alloc <= brickinfo->port)
-                                        pmap->last_alloc = brickinfo->port + 1;
+
+                                if (brickinfo->port < priv->base_port) {
+                                        /* This is required to adhere to the
+                                           IANA standards */
+                                        brickinfo->port = 0;
+                                } else {
+                                        /* This is required to have proper ports
+                                           assigned to bricks after restart */
+                                        pmap = pmap_registry_get (THIS);
+                                        if (pmap->last_alloc <= brickinfo->port)
+                                                pmap->last_alloc =
+                                                        brickinfo->port + 1;
+                                }
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
                                     strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
                                 gf_string2int (value, &brickinfo->rdma_port);
-                                /* This is required to have proper ports
-                                   assigned to bricks after restart */
-                                pmap = pmap_registry_get (THIS);
-                                if (pmap->last_alloc <= brickinfo->rdma_port)
-                                        pmap->last_alloc = brickinfo->rdma_port + 1;
+
+                                if (brickinfo->rdma_port < priv->base_port) {
+                                        /* This is required to adhere to the
+                                           IANA standards */
+                                        brickinfo->rdma_port = 0;
+                                } else {
+                                        /* This is required to have proper ports
+                                           assigned to bricks after restart */
+                                        pmap = pmap_registry_get (THIS);
+                                        if (pmap->last_alloc <=
+                                            brickinfo->rdma_port)
+                                                pmap->last_alloc =
+                                                        brickinfo->rdma_port +1;
+                                }
+
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
                                              strlen (GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
                                 gf_string2int (value, &brickinfo->decommissioned);
+                        } else if (!strncmp (key,
+                                    GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
+                                strncpy (brickinfo->vg, value,
+                                         sizeof (brickinfo->vg));
+                        } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
+                                strncpy (brickinfo->brick_id, value,
+                                         sizeof (brickinfo->brick_id));
                         } else {
                                 gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
                                         key);
@@ -1689,22 +1624,31 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         key = NULL;
                         value = NULL;
 
-                        ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                            &op_errno);
+                        ret = gf_store_iter_get_next (iter, &key, &value,
+                                                      &op_errno);
                 }
 
-                if (op_errno != GD_STORE_EOF)
+                if (op_errno != GD_STORE_EOF) {
+                        gf_log ("", GF_LOG_ERROR, "Error parsing brickinfo: "
+                                "op_errno=%d", op_errno);
                         goto out;
-                ret = glusterd_store_iter_destroy (iter);
+                }
+                ret = gf_store_iter_destroy (iter);
 
                 if (ret)
                         goto out;
 
+                if (brickinfo->brick_id[0] == '\0') {
+                        /* This is an old volume upgraded to op_version 4 */
+                       GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo,
+                                                             brickid++);
+                }
+
                 list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
                 brick_count++;
         }
 
-        ret = glusterd_store_iter_destroy (tmpiter);
+        ret = gf_store_iter_destroy (tmpiter);
         if (ret)
                 goto out;
 out:
@@ -1719,13 +1663,13 @@ glusterd_store_retrieve_rbstate (char   *volname)
 {
         int32_t                   ret                   = -1;
         glusterd_volinfo_t        *volinfo              = NULL;
-        glusterd_store_iter_t     *iter                 = NULL;
+        gf_store_iter_t           *iter                 = NULL;
         char                      *key                  = NULL;
         char                      *value                = NULL;
         char                      volpath[PATH_MAX]     = {0,};
         glusterd_conf_t           *priv                 = NULL;
         char                      path[PATH_MAX]        = {0,};
-        glusterd_store_op_errno_t op_errno              = GD_STORE_SUCCESS;
+        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
 
         priv = THIS->private;
 
@@ -1740,39 +1684,57 @@ glusterd_store_retrieve_rbstate (char   *volname)
         snprintf (path, sizeof (path), "%s/%s", volpath,
                   GLUSTERD_VOLUME_RBSTATE_FILE);
 
-        ret = glusterd_store_handle_retrieve (path, &volinfo->rb_shandle);
+        ret = gf_store_handle_retrieve (path, &volinfo->rb_shandle);
 
         if (ret)
                 goto out;
 
-        ret = glusterd_store_iter_new (volinfo->rb_shandle, &iter);
+        ret = gf_store_iter_new (volinfo->rb_shandle, &iter);
 
         if (ret)
                 goto out;
 
-        ret = glusterd_store_iter_get_next (iter, &key, &value, &op_errno);
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
         if (ret)
                 goto out;
 
         while (!ret) {
                 if (!strncmp (key, GLUSTERD_STORE_KEY_RB_STATUS,
                               strlen (GLUSTERD_STORE_KEY_RB_STATUS))) {
-                        volinfo->rb_status = atoi (value);
+                        volinfo->rep_brick.rb_status = atoi (value);
                 }
 
-                if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+                if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
                         if (!strncmp (key, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
                                       strlen (GLUSTERD_STORE_KEY_RB_SRC_BRICK))) {
-                                ret = glusterd_brickinfo_from_brick (value,
-                                                                     &volinfo->src_brick);
+                                ret = glusterd_brickinfo_new_from_brick (value,
+                                                &volinfo->rep_brick.src_brick);
                                 if (ret)
                                         goto out;
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_BRICK,
                                              strlen (GLUSTERD_STORE_KEY_RB_DST_BRICK))) {
-                                ret = glusterd_brickinfo_from_brick (value,
-                                                                     &volinfo->dst_brick);
+                                ret = glusterd_brickinfo_new_from_brick (value,
+                                                &volinfo->rep_brick.dst_brick);
                                 if (ret)
                                         goto out;
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_PORT,
+                                             strlen (GLUSTERD_STORE_KEY_RB_DST_PORT))) {
+                                switch (volinfo->transport_type) {
+                                case GF_TRANSPORT_RDMA:
+                                volinfo->rep_brick.dst_brick->rdma_port =
+                                                 atoi (value);
+                                        break;
+
+                                case GF_TRANSPORT_TCP:
+                                case GF_TRANSPORT_BOTH_TCP_RDMA:
+                                volinfo->rep_brick.dst_brick->port =
+                                                atoi (value);
+                                        break;
+                                }
+                        } else if (!strncmp (key, GF_REPLACE_BRICK_TID_KEY,
+                                             strlen (GF_REPLACE_BRICK_TID_KEY))) {
+                                        uuid_parse (value,
+                                                    volinfo->rep_brick.rb_id);
                         }
                 }
 
@@ -1781,19 +1743,129 @@ glusterd_store_retrieve_rbstate (char   *volname)
                 key = NULL;
                 value = NULL;
 
-                ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                    &op_errno);
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
         }
 
         if (op_errno != GD_STORE_EOF)
                 goto out;
 
-        ret = glusterd_store_iter_destroy (iter);
+        ret = gf_store_iter_destroy (iter);
+
+        if (ret)
+                goto out;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_node_state (char   *volname)
+{
+        int32_t              ret               = -1;
+        glusterd_volinfo_t  *volinfo           = NULL;
+        gf_store_iter_t     *iter              = NULL;
+        char                *key               = NULL;
+        char                *value             = NULL;
+        char                *dup_value         = NULL;
+        char                 volpath[PATH_MAX] = {0,};
+        glusterd_conf_t     *priv              = NULL;
+        char                 path[PATH_MAX]    = {0,};
+        gf_store_op_errno_t  op_errno          = GD_STORE_SUCCESS;
+        dict_t              *tmp_dict          = NULL;
+        xlator_t            *this              = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get"
+                        "volinfo for %s.", volname);
+                goto out;
+        }
+
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
+        snprintf (path, sizeof (path), "%s/%s", volpath,
+                  GLUSTERD_NODE_STATE_FILE);
+
+        ret = gf_store_handle_retrieve (path, &volinfo->node_state_shandle);
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_new (volinfo->node_state_shandle, &iter);
+
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret)
+                goto out;
+
+        while (ret == 0) {
+                if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
+                              strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
+                        volinfo->rebal.defrag_cmd = atoi (value);
+                } else if (!strncmp (key, GF_REBALANCE_TID_KEY,
+                                     strlen (GF_REBALANCE_TID_KEY))) {
+                        uuid_parse (value, volinfo->rebal.rebalance_id);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
+                                        strlen (GLUSTERD_STORE_KEY_DEFRAG_OP))) {
+                        volinfo->rebal.op = atoi (value);
+                } else {
+                        if (!tmp_dict) {
+                                tmp_dict = dict_new ();
+                                if (!tmp_dict) {
+                                        ret = -1;
+                                        goto out;
+                                }
+                        }
+                        dup_value = gf_strdup (value);
+                        if (!dup_value) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to strdup value string");
+                                goto out;
+                        }
+                        ret = dict_set_str (tmp_dict, key, dup_value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                                "Error setting data in rebal "
+                                                "dict.");
+                                goto out;
+                        }
+                        dup_value = NULL;
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+        if (tmp_dict)
+                volinfo->rebal.dict = dict_ref (tmp_dict);
+
+        if (op_errno != GD_STORE_EOF) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = gf_store_iter_destroy (iter);
 
         if (ret)
                 goto out;
 
 out:
+        if (dup_value)
+                GF_FREE (dup_value);
+        if (ret && volinfo->rebal.dict)
+                dict_unref (volinfo->rebal.dict);
+        if (tmp_dict)
+                dict_unref (tmp_dict);
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
@@ -1804,17 +1876,16 @@ glusterd_store_retrieve_volume (char    *volname)
 {
         int32_t                   ret                   = -1;
         glusterd_volinfo_t        *volinfo              = NULL;
-        glusterd_store_iter_t     *iter                 = NULL;
+        gf_store_iter_t           *iter                 = NULL;
         char                      *key                  = NULL;
         char                      *value                = NULL;
         char                      volpath[PATH_MAX]     = {0,};
         glusterd_conf_t           *priv                 = NULL;
         char                      path[PATH_MAX]        = {0,};
         int                       exists                = 0;
-        glusterd_store_op_errno_t op_errno              = GD_STORE_SUCCESS;
+        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
 
         ret = glusterd_volinfo_new (&volinfo);
-
         if (ret)
                 goto out;
 
@@ -1826,17 +1897,15 @@ glusterd_store_retrieve_volume (char    *volname)
         snprintf (path, sizeof (path), "%s/%s", volpath,
                   GLUSTERD_VOLUME_INFO_FILE);
 
-        ret = glusterd_store_handle_retrieve (path, &volinfo->shandle);
-
+        ret = gf_store_handle_retrieve (path, &volinfo->shandle);
         if (ret)
                 goto out;
 
-        ret = glusterd_store_iter_new (volinfo->shandle, &iter);
-
+        ret = gf_store_iter_new (volinfo->shandle, &iter);
         if (ret)
                 goto out;
 
-        ret = glusterd_store_iter_get_next (iter, &key, &value, &op_errno);
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
         if (ret)
                 goto out;
 
@@ -1899,31 +1968,59 @@ glusterd_store_retrieve_volume (char    *volname)
                         }
                         gf_log ("", GF_LOG_DEBUG, "Parsed as "GEOREP" "
                                 " slave:key=%s,value:%s", key, value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
-                        volinfo->defrag_cmd = atoi (value);
-                }
-                else {
-                        exists = glusterd_check_option_exists (key, NULL);
-                        if (exists == -1) {
+
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
+                                strlen (GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
+                        volinfo->op_version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                                strlen (GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
+                        volinfo->client_op_version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CAPS,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_CAPS))) {
+                        volinfo->caps = atoi (value);
+                } else {
+
+                        if (is_key_glusterd_hooks_friendly (key)) {
+                                exists = 1;
+
+                        } else  {
+                                exists = glusterd_check_option_exists (key,
+                                                                       NULL);
+                        }
+
+                        switch (exists) {
+                        case -1:
                                 ret = -1;
                                 goto out;
-                        }
-                        if (exists) {
+
+                        case 0:
+                                 /*Ignore GLUSTERD_STORE_KEY_VOL_BRICK since
+                                  glusterd_store_retrieve_bricks gets it later*/
+                                if (!strstr (key, GLUSTERD_STORE_KEY_VOL_BRICK))
+                                        gf_log ("", GF_LOG_WARNING,
+                                                "Unknown key: %s", key);
+                                break;
+
+                        case 1:
+                                /*The following strcmp check is to ensure that
+                                 * glusterd does not restore the quota limits
+                                 * into volinfo->dict post upgradation from 3.3
+                                 * to 3.4 as the same limits will now be stored
+                                 * in xattrs on the respective directories.
+                                 */
+                                if (!strcmp (key, "features.limit-usage"))
+                                        break;
                                 ret = dict_set_str(volinfo->dict, key,
-                                                     gf_strdup (value));
+                                                   gf_strdup (value));
                                 if (ret) {
                                         gf_log ("",GF_LOG_ERROR, "Error in "
-                                                        "dict_set_str");
+                                                "dict_set_str");
                                         goto out;
                                 }
                                 gf_log ("", GF_LOG_DEBUG, "Parsed as Volume-"
-                                                "set:key=%s,value:%s",
-                                                                key, value);
+                                        "set:key=%s,value:%s", key, value);
+                                break;
                         }
-                        else
-                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
-                                        key);
                 }
 
                 GF_FREE (key);
@@ -1931,8 +2028,7 @@ glusterd_store_retrieve_volume (char    *volname)
                 key = NULL;
                 value = NULL;
 
-                ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                    &op_errno);
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
         }
 
         /* backward compatibility */
@@ -1966,14 +2062,20 @@ glusterd_store_retrieve_volume (char    *volname)
                         break;
                 }
 
-                volinfo->dist_leaf_count = (volinfo->stripe_count *
-                                            volinfo->replica_count);
+                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+
+                volinfo->subvol_count = (volinfo->brick_count /
+                                         volinfo->dist_leaf_count);
+
+                /* Only calculate volume op-versions if they are not found */
+                if (!volinfo->op_version && !volinfo->client_op_version)
+                        gd_update_volume_op_versions (volinfo);
         }
 
         if (op_errno != GD_STORE_EOF)
                 goto out;
 
-        ret = glusterd_store_iter_destroy (iter);
+        ret = gf_store_iter_destroy (iter);
 
         if (ret)
                 goto out;
@@ -1982,11 +2084,29 @@ glusterd_store_retrieve_volume (char    *volname)
         if (ret)
                 goto out;
 
-        ret = glusterd_volume_compute_cksum (volinfo);
+        ret = glusterd_compute_cksum (volinfo, _gf_false);
         if (ret)
                 goto out;
 
-        list_add_tail (&volinfo->vol_list, &priv->volumes);
+        ret = glusterd_store_retrieve_quota_version (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_create_quota_conf_sh_on_absence (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_compute_cksum (volinfo, _gf_true);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_save_quota_version_and_cksum (volinfo);
+        if (ret)
+                goto out;
+
+
+        list_add_order (&volinfo->vol_list, &priv->volumes,
+                         glusterd_compare_volume_name);
 
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
@@ -1994,6 +2114,101 @@ out:
         return ret;
 }
 
+inline void
+glusterd_store_set_options_path (glusterd_conf_t *conf, char *path, size_t len)
+{
+        snprintf (path, len, "%s/options", conf->workdir);
+}
+
+int
+_store_global_opts (dict_t *this, char *key, data_t *value, void *data)
+{
+        gf_store_handle_t *shandle = data;
+
+        gf_store_save_value (shandle->fd, key, (char*)value->data);
+        return 0;
+}
+
+int32_t
+glusterd_store_options (xlator_t *this, dict_t *opts)
+{
+        gf_store_handle_t               *shandle = NULL;
+        glusterd_conf_t                 *conf = NULL;
+        char                            path[PATH_MAX] = {0};
+        int                             fd = -1;
+        int32_t                         ret = -1;
+
+        conf = this->private;
+        glusterd_store_set_options_path (conf, path, sizeof (path));
+
+        ret = gf_store_handle_new (path, &shandle);
+        if (ret)
+                goto out;
+
+        fd = gf_store_mkstemp (shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        shandle->fd = fd;
+        dict_foreach (opts, _store_global_opts, shandle);
+        shandle->fd = 0;
+        ret = gf_store_rename_tmppath (shandle);
+        if (ret)
+                goto out;
+out:
+        gf_store_handle_destroy (shandle);
+        if (fd >=0 )
+                close (fd);
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_options (xlator_t *this)
+{
+        char                            path[PATH_MAX] = {0};
+        glusterd_conf_t                 *conf = NULL;
+        gf_store_handle_t               *shandle = NULL;
+        gf_store_iter_t                 *iter = NULL;
+        char                            *key = NULL;
+        char                            *value = NULL;
+        gf_store_op_errno_t             op_errno = 0;
+        int                             ret = -1;
+
+        conf = this->private;
+        glusterd_store_set_options_path (conf, path, sizeof (path));
+
+        ret = gf_store_handle_retrieve (path, &shandle);
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_new (shandle, &iter);
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        while (!ret) {
+                ret = dict_set_dynstr (conf->opts, key, value);
+                if (ret) {
+                        GF_FREE (key);
+                        GF_FREE (value);
+                        goto out;
+                }
+                GF_FREE (key);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+        ret = 0;
+out:
+        gf_store_iter_destroy (iter);
+        gf_store_handle_destroy (shandle);
+        return ret;
+}
 
 int32_t
 glusterd_store_retrieve_volumes (xlator_t  *this)
@@ -2040,6 +2255,18 @@ glusterd_store_retrieve_volumes (xlator_t  *this)
                         ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
                         ret = glusterd_store_perform_rbstate_store (volinfo);
                 }
+
+                ret = glusterd_store_retrieve_node_state (entry->d_name);
+                if (ret) {
+                        /* Backward compatibility */
+                        gf_log ("", GF_LOG_INFO, "Creating a new node_state "
+                                "for volume: %s.", entry->d_name);
+                        ret = glusterd_volinfo_find (entry->d_name, &volinfo);
+                        ret =
+                        glusterd_store_create_nodestate_sh_on_absence (volinfo);
+                        ret = glusterd_store_perform_node_state_store (volinfo);
+
+                }
                 glusterd_for_each_entry (entry, dir);
         }
 
@@ -2099,7 +2326,7 @@ glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
 
 out:
         if (peerinfo->shandle) {
-                glusterd_store_handle_destroy (peerinfo->shandle);
+                gf_store_handle_destroy (peerinfo->shandle);
                 peerinfo->shandle = NULL;
         }
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
@@ -2125,7 +2352,7 @@ glusterd_store_create_peer_dir ()
         char                            path[PATH_MAX];
 
         glusterd_store_peerinfo_dirpath_set (path, sizeof (path));
-        ret = glusterd_store_mkdir (path);
+        ret = gf_store_mkdir (path);
 
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
@@ -2169,8 +2396,8 @@ glusterd_store_peerinfo_hostname_shandle_create (glusterd_peerinfo_t *peerinfo)
 
         glusterd_store_hostname_peerpath_set (peerinfo, peerfpath,
                                               sizeof (peerfpath));
-        ret = glusterd_store_handle_create_on_absence (&peerinfo->shandle,
-                                                       peerfpath);
+        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
+                                                 peerfpath);
         return ret;
 }
 
@@ -2182,8 +2409,8 @@ glusterd_store_peerinfo_uuid_shandle_create (glusterd_peerinfo_t *peerinfo)
 
         glusterd_store_uuid_peerpath_set (peerinfo, peerfpath,
                                           sizeof (peerfpath));
-        ret = glusterd_store_handle_create_on_absence (&peerinfo->shandle,
-                                                       peerfpath);
+        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
+                                                 peerfpath);
         return ret;
 }
 
@@ -2199,7 +2426,7 @@ glusterd_peerinfo_hostname_shandle_check_destroy (glusterd_peerinfo_t *peerinfo)
         ret = stat (peerfpath, &stbuf);
         if (!ret) {
                 if (peerinfo->shandle)
-                        glusterd_store_handle_destroy (peerinfo->shandle);
+                        gf_store_handle_destroy (peerinfo->shandle);
                 peerinfo->shandle = NULL;
                 ret = unlink (peerfpath);
         }
@@ -2228,18 +2455,21 @@ glusterd_store_peer_write (int fd, glusterd_peerinfo_t *peerinfo)
         char                    buf[50] = {0};
         int32_t                 ret = 0;
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_UUID,
-                                         uuid_utoa (peerinfo->uuid));
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_UUID,
+                                   uuid_utoa (peerinfo->uuid));
         if (ret)
                 goto out;
 
         snprintf (buf, sizeof (buf), "%d", peerinfo->state.state);
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_STATE, buf);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_STATE, buf);
+        if (ret)
+                goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_HOSTNAME "1",
+                                   peerinfo->hostname);
         if (ret)
                 goto out;
 
-        ret = glusterd_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_HOSTNAME "1",
-                                         peerinfo->hostname);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
@@ -2253,7 +2483,7 @@ glusterd_store_perform_peer_store (glusterd_peerinfo_t *peerinfo)
 
         GF_ASSERT (peerinfo);
 
-        fd = glusterd_store_mkstemp (peerinfo->shandle);
+        fd = gf_store_mkstemp (peerinfo->shandle);
         if (fd <= 0) {
                 ret = -1;
                 goto out;
@@ -2263,10 +2493,10 @@ glusterd_store_perform_peer_store (glusterd_peerinfo_t *peerinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_store_rename_tmppath (peerinfo->shandle);
+        ret = gf_store_rename_tmppath (peerinfo->shandle);
 out:
         if (ret && (fd > 0))
-                glusterd_store_unlink_tmppath (peerinfo->shandle);
+                gf_store_unlink_tmppath (peerinfo->shandle);
         if (fd > 0)
                 close (fd);
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
@@ -2306,13 +2536,13 @@ glusterd_store_retrieve_peers (xlator_t *this)
         uuid_t                  uuid = {0,};
         char                    *hostname = NULL;
         int32_t                 state = 0;
-        glusterd_store_handle_t *shandle = NULL;
+        gf_store_handle_t       *shandle = NULL;
         char                    filepath[PATH_MAX] = {0,};
-        glusterd_store_iter_t   *iter = NULL;
+        gf_store_iter_t         *iter = NULL;
         char                    *key = NULL;
         char                    *value = NULL;
         glusterd_peerctx_args_t args = {0};
-        glusterd_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+        gf_store_op_errno_t     op_errno = GD_STORE_SUCCESS;
 
         GF_ASSERT (this);
         priv = this->private;
@@ -2334,16 +2564,15 @@ glusterd_store_retrieve_peers (xlator_t *this)
 
         while (entry) {
                 snprintf (filepath, PATH_MAX, "%s/%s", path, entry->d_name);
-                ret = glusterd_store_handle_retrieve (filepath, &shandle);
+                ret = gf_store_handle_retrieve (filepath, &shandle);
                 if (ret)
                         goto out;
 
-                ret = glusterd_store_iter_new (shandle, &iter);
+                ret = gf_store_iter_new (shandle, &iter);
                 if (ret)
                         goto out;
 
-                ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                    &op_errno);
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
                 if (ret)
                         goto out;
 
@@ -2371,17 +2600,18 @@ glusterd_store_retrieve_peers (xlator_t *this)
                         key = NULL;
                         value = NULL;
 
-                        ret = glusterd_store_iter_get_next (iter, &key, &value,
-                                                            &op_errno);
+                        ret = gf_store_iter_get_next (iter, &key, &value,
+                                                      &op_errno);
                 }
-                if (op_errno != GD_STORE_EOF)
+                if (op_errno != GD_STORE_EOF) {
+                        GF_FREE(hostname);
                         goto out;
+                }
 
-                (void) glusterd_store_iter_destroy (iter);
+                (void) gf_store_iter_destroy (iter);
 
-                args.mode = GD_MODE_SWITCH_ON;
                 ret = glusterd_friend_add (hostname, 0, state, &uuid,
-                                           &peerinfo, 1, &args);
+                                           &peerinfo, 1, NULL);
 
                 GF_FREE (hostname);
                 if (ret)
@@ -2391,6 +2621,13 @@ glusterd_store_retrieve_peers (xlator_t *this)
                 glusterd_for_each_entry (entry, dir);
         }
 
+        args.mode = GD_MODE_ON;
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                ret = glusterd_friend_rpc_create (this, peerinfo, &args);
+                if (ret)
+                        goto out;
+        }
+
 out:
         if (dir)
                 closedir (dir);
@@ -2437,8 +2674,14 @@ glusterd_restore ()
 
         this = THIS;
 
-        ret = glusterd_store_retrieve_volumes (this);
+        ret = glusterd_restore_op_version (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to restore op_version");
+                goto out;
+        }
 
+        ret = glusterd_store_retrieve_volumes (this);
         if (ret)
                 goto out;
 
@@ -2454,3 +2697,106 @@ out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+
+int
+glusterd_store_retrieve_quota_version (glusterd_volinfo_t *volinfo)
+{
+        int                 ret                  = -1;
+        uint32_t            version              = 0;
+        char                cksum_path[PATH_MAX] = {0,};
+        char                path[PATH_MAX]       = {0,};
+        char               *version_str          = NULL;
+        char               *tmp                  = NULL;
+        xlator_t           *this                 = NULL;
+        glusterd_conf_t    *conf                 = NULL;
+        gf_store_handle_t  *handle               = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
+        snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
+                  GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+
+        ret = gf_store_handle_new (cksum_path, &handle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get store handle "
+                        "for %s", cksum_path);
+                goto out;
+        }
+
+        ret = gf_store_retrieve_value (handle, "version", &version_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Version absent");
+                ret = 0;
+                goto out;
+        }
+
+        version = strtoul (version_str, &tmp, 10);
+         if (version < 0) {
+                 gf_log (this->name, GF_LOG_DEBUG, "Invalid version number");
+                 goto out;
+        }
+        volinfo->quota_conf_version = version;
+        ret = 0;
+
+out:
+        if (version_str)
+                GF_FREE (version_str);
+        gf_store_handle_destroy (handle);
+        return ret;
+}
+
+int
+glusterd_store_save_quota_version_and_cksum (glusterd_volinfo_t *volinfo)
+{
+        int                 ret                  = -1;
+        char                cksum_path[PATH_MAX] = {0,};
+        char                path[PATH_MAX]       = {0,};
+        xlator_t           *this                 = NULL;
+        glusterd_conf_t    *conf                 = NULL;
+        char                buf[256]             = {0,};
+        int                 fd                   = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
+        snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
+                  GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+
+        fd = open (cksum_path, O_RDWR | O_APPEND | O_CREAT| O_TRUNC, 0600);
+
+        if (-1 == fd) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to open %s,"
+                        "Reason: %s", cksum_path, strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf)-1, "%u", volinfo->quota_conf_cksum);
+        ret = gf_store_save_value (fd, "cksum", buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store cksum");
+                goto out;
+        }
+
+        memset (buf, 0, sizeof (buf));
+        snprintf (buf, sizeof (buf)-1, "%u", volinfo->quota_conf_version);
+        ret = gf_store_save_value (fd, "version", buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store version");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (fd != -1)
+                close (fd);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index b381e5a0c..955abb09f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_HA_H_
 #define _GLUSTERD_HA_H_
 
@@ -30,6 +20,7 @@
 
 #include "glusterfs.h"
 #include "xlator.h"
+#include "run.h"
 #include "logging.h"
 #include "call-stub.h"
 #include "fd.h"
@@ -40,6 +31,7 @@
 typedef enum glusterd_store_ver_ac_{
         GLUSTERD_VOLINFO_VER_AC_NONE = 0,
         GLUSTERD_VOLINFO_VER_AC_INCREMENT = 1,
+        GLUSTERD_VOLINFO_VER_AC_DECREMENT = 2,
 } glusterd_volinfo_ver_ac_t;
 
 
@@ -59,41 +51,41 @@ typedef enum glusterd_store_ver_ac_{
 #define GLUSTERD_STORE_KEY_RB_STATUS      "rb_status"
 #define GLUSTERD_STORE_KEY_RB_SRC_BRICK   "rb_src"
 #define GLUSTERD_STORE_KEY_RB_DST_BRICK   "rb_dst"
+#define GLUSTERD_STORE_KEY_RB_DST_PORT    "rb_port"
 #define GLUSTERD_STORE_KEY_VOL_DEFRAG     "rebalance_status"
+#define GLUSTERD_STORE_KEY_DEFRAG_OP      "rebalance_op"
 #define GLUSTERD_STORE_KEY_USERNAME       "username"
 #define GLUSTERD_STORE_KEY_PASSWORD       "password"
+#define GLUSTERD_STORE_KEY_VOL_OP_VERSION "op-version"
+#define GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION "client-op-version"
 
 #define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
 #define GLUSTERD_STORE_KEY_BRICK_PATH     "path"
 #define GLUSTERD_STORE_KEY_BRICK_PORT     "listen-port"
 #define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port"
 #define GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED "decommissioned"
+#define GLUSTERD_STORE_KEY_BRICK_VGNAME "vg"
+#define GLUSTERD_STORE_KEY_BRICK_ID "brick-id"
 
 #define GLUSTERD_STORE_KEY_PEER_UUID      "uuid"
 #define GLUSTERD_STORE_KEY_PEER_HOSTNAME  "hostname"
 #define GLUSTERD_STORE_KEY_PEER_STATE     "state"
 
+#define GLUSTERD_STORE_KEY_VOL_CAPS       "caps"
+
 #define glusterd_for_each_entry(entry, dir) \
         do {\
                 entry = NULL;\
                 if (dir) {\
                         entry = readdir (dir);\
                         while (entry && (!strcmp (entry->d_name, ".") ||\
+                            !fnmatch ("*.tmp", entry->d_name, 0) ||\
                             !strcmp (entry->d_name, ".."))) {\
                                 entry = readdir (dir);\
                         }\
                 }\
         } while (0); \
 
-typedef enum {
-        GD_STORE_SUCCESS,
-        GD_STORE_KEY_NULL,
-        GD_STORE_VALUE_NULL,
-        GD_STORE_KEY_VALUE_NULL,
-        GD_STORE_EOF,
-        GD_STORE_ENOMEM,
-        GD_STORE_STAT_FAILED
-} glusterd_store_op_errno_t;
 
 int32_t
 glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac);
@@ -102,19 +94,6 @@ int32_t
 glusterd_store_delete_volume (glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_store_uuid ();
-
-int32_t
-glusterd_store_handle_new (char *path, glusterd_store_handle_t **handle);
-
-int32_t
-glusterd_store_save_value (int fd, char *key, char *value);
-
-int32_t
-glusterd_store_retrieve_value (glusterd_store_handle_t *handle,
-                               char *key, char **value);
-
-int32_t
 glusterd_retrieve_uuid ();
 
 int32_t
@@ -124,11 +103,8 @@ int32_t
 glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo);
 
 int32_t
-glusterd_store_delete_brick (glusterd_volinfo_t *volinfo,
-                             glusterd_brickinfo_t *brickinfo);
-
-int32_t
-glusterd_store_handle_destroy (glusterd_store_handle_t *handle);
+glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo,
+                             char *delete_path);
 
 int32_t
 glusterd_restore ();
@@ -138,4 +114,29 @@ glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
                                          glusterd_volinfo_ver_ac_t ac);
 gf_boolean_t
 glusterd_store_is_valid_brickpath (char *volname, char *brick);
+
+int32_t
+glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_retrieve_op_version (xlator_t *this, int *op_version);
+
+int
+glusterd_store_global_info (xlator_t *this);
+
+int32_t
+glusterd_store_retrieve_options (xlator_t *this);
+
+int32_t
+glusterd_store_options (xlator_t *this, dict_t *opts);
+
+int32_t
+glusterd_store_create_quota_conf_sh_on_absence (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_store_retrieve_quota_version (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_store_save_quota_version_and_cksum (glusterd_volinfo_t *volinfo);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
new file mode 100644
index 000000000..578bce897
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -0,0 +1,1690 @@
+/*
+   Copyright (c) 2012-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+
+extern glusterd_op_info_t opinfo;
+
+static inline void
+gd_synctask_barrier_wait (struct syncargs *args, int count)
+{
+        glusterd_conf_t *conf = THIS->private;
+
+        synclock_unlock (&conf->big_lock);
+        synctask_barrier_wait (args, count);
+        synclock_lock (&conf->big_lock);
+
+	syncbarrier_destroy (&args->barrier);
+}
+
+static void
+gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                           char *op_errstr, int op_code,
+                           glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char     err_str[PATH_MAX] = "Please check log file for details.";
+        char     op_err[PATH_MAX] = "";
+        char    *peer_str      = NULL;
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, ""))
+                        snprintf (err_str, sizeof(err_str) - 1,
+                                  "Error: %s", op_errstr);
+
+                switch (op_code) {
+                        case GLUSTERD_MGMT_V3_VOLUME_LOCK:
+                        {
+                                snprintf (op_err, sizeof(op_err) - 1,
+                                          "Locking volume failed "
+                                          "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_V3_VOLUME_UNLOCK:
+                        {
+                                snprintf (op_err, sizeof(op_err) - 1,
+                                          "Unlocking volume failed "
+                                          "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                }
+
+                if (args->errstr) {
+                        snprintf (err_str, sizeof(err_str) - 1,
+                                  "%s\n%s", args->errstr,
+                                  op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        snprintf (err_str, sizeof(err_str) - 1,
+                                  "%s", op_err);
+
+                gf_log ("", GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+static void
+gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                   char *op_errstr, int op_code,
+                   glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char     err_str[PATH_MAX] = "Please check log file for details.";
+        char     op_err[PATH_MAX] = "";
+        int      len               = -1;
+        char    *peer_str      = NULL;
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
+                        case GLUSTERD_MGMT_CLUSTER_LOCK :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Locking failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_CLUSTER_UNLOCK :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_STAGE_OP :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Staging failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_COMMIT_OP :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Commit failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log ("", GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+static void
+gd_syncargs_init (struct syncargs *args, dict_t *op_ctx)
+{
+        args->dict = op_ctx;
+        pthread_mutex_init (&args->lock_dict, NULL);
+}
+
+static void
+gd_stage_op_req_free (gd1_mgmt_stage_op_req *req)
+{
+        if (!req)
+                return;
+
+        GF_FREE (req->buf.buf_val);
+        GF_FREE (req);
+}
+
+static void
+gd_commit_op_req_free (gd1_mgmt_commit_op_req *req)
+{
+        if (!req)
+                return;
+
+        GF_FREE (req->buf.buf_val);
+        GF_FREE (req);
+}
+
+static void
+gd_brick_op_req_free (gd1_mgmt_brick_op_req *req)
+{
+        if (!req)
+                return;
+
+        if (strcmp (req->name, "") != 0)
+                GF_FREE (req->name);
+        GF_FREE (req->input.input_val);
+        GF_FREE (req);
+}
+
+int
+gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
+                          void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+        int            ret      = -1;
+        struct iobuf  *iobuf    = NULL;
+        struct iobref *iobref   = NULL;
+        int            count    = 0;
+        struct iovec   iov      = {0, };
+        ssize_t        req_size = 0;
+        call_frame_t  *frame    = NULL;
+
+        GF_ASSERT (rpc);
+        if (!req)
+                goto out;
+
+        req_size = xdr_sizeof (xdrproc, req);
+        iobuf = iobuf_get2 (rpc->ctx->iobuf_pool, req_size);
+        if (!iobuf)
+                goto out;
+
+        iobref = iobref_new ();
+        if (!iobref)
+                goto out;
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        iobref_add (iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len  = iobuf_pagesize (iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic (iov, req, xdrproc);
+        if (ret == -1)
+                goto out;
+
+        iov.iov_len = ret;
+        count = 1;
+
+        frame->local = local;
+        frame->cookie = cookie;
+
+        /* Send the msg */
+        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
+                               &iov, count, NULL, 0, iobref,
+                               frame, NULL, 0, NULL, 0, NULL);
+
+        /* TODO: do we need to start ping also? */
+
+out:
+        iobref_unref (iobref);
+        iobuf_unref (iobuf);
+
+        return ret;
+}
+
+/* Defined in glusterd-rpc-ops.c */
+extern struct rpc_clnt_program gd_mgmt_prog;
+extern struct rpc_clnt_program gd_brick_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
+static int
+glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
+{
+        int ret = 0;
+
+        switch (op) {
+        case GD_OP_REPLACE_BRICK:
+                ret = glusterd_rb_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_SYNC_VOLUME:
+                ret = glusterd_sync_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_GSYNC_CREATE:
+        break;
+
+        case GD_OP_GSYNC_SET:
+                ret = glusterd_gsync_use_rsp_dict (aggr, rsp, NULL);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_STATUS_VOLUME:
+                ret = glusterd_volume_status_copy_to_op_ctx_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = glusterd_volume_rebalance_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_HEAL_VOLUME:
+                ret = glusterd_volume_heal_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+
+        break;
+
+        case GD_OP_CLEARLOCKS_VOLUME:
+                ret = glusterd_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_QUOTA:
+                ret = glusterd_volume_quota_copy_to_op_ctx_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+                break;
+
+        case GD_OP_SYS_EXEC:
+                ret = glusterd_sys_exec_output_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        default:
+        break;
+        }
+out:
+        return ret;
+}
+
+int32_t
+_gd_syncop_mgmt_volume_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_volume_lock_rsp     rsp        = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        GF_ASSERT(req);
+        GF_ASSERT(iov);
+        GF_ASSERT(myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_volume_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_VOLUME_LOCK,
+                                   peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_volume_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_volume_lock_cbk);
+}
+
+int
+gd_syncop_mgmt_volume_lock (glusterd_op_t op, dict_t *op_ctx,
+                            glusterd_peerinfo_t *peerinfo,
+                            struct syncargs *args, uuid_t my_uuid,
+                            uuid_t recv_uuid, uuid_t txn_id)
+{
+        int                      ret  = -1;
+        gd1_mgmt_volume_lock_req req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+
+        GF_ASSERT(op_ctx);
+        GF_ASSERT(peerinfo);
+        GF_ASSERT(args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        uuid_copy (req.txn_id, txn_id);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_VOLUME_LOCK,
+                                        gd_syncop_mgmt_volume_lock_cbk,
+                                        (xdrproc_t)
+                                        xdr_gd1_mgmt_volume_lock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+_gd_syncop_mgmt_volume_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                                   int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_volume_unlock_rsp   rsp        = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        GF_ASSERT(req);
+        GF_ASSERT(iov);
+        GF_ASSERT(myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_volume_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        /* Set peer as locked, so we unlock only the locked peers */
+        if (rsp.op_ret == 0)
+                peerinfo->locked = _gf_true;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_VOLUME_UNLOCK,
+                                   peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_volume_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                                  int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_volume_unlock_cbk);
+}
+
+int
+gd_syncop_mgmt_volume_unlock (dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
+                              struct syncargs *args, uuid_t my_uuid,
+                              uuid_t recv_uuid, uuid_t txn_id)
+{
+        int                          ret  = -1;
+        gd1_mgmt_volume_unlock_req   req  = {{0},};
+        glusterd_conf_t             *conf = THIS->private;
+
+        GF_ASSERT(op_ctx);
+        GF_ASSERT(peerinfo);
+        GF_ASSERT(args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        uuid_copy (req.txn_id, txn_id);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_VOLUME_UNLOCK,
+                                        gd_syncop_mgmt_volume_unlock_cbk,
+                                        (xdrproc_t)
+                                        xdr_gd1_mgmt_volume_unlock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+_gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_cluster_lock_rsp   rsp         = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        /* Set peer as locked, so we unlock only the locked peers */
+        if (rsp.op_ret == 0)
+                peerinfo->locked = _gf_true;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_collate_errors (args, op_ret, op_errno, NULL,
+                           GLUSTERD_MGMT_CLUSTER_LOCK, peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_lock_cbk);
+}
+
+int
+gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                     uuid_t my_uuid, uuid_t recv_uuid)
+{
+        int                       ret = -1;
+        gd1_mgmt_cluster_lock_req req  = {{0},};
+        glusterd_conf_t           *conf = THIS->private;
+
+        uuid_copy (req.uuid, my_uuid);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_CLUSTER_LOCK,
+                                        gd_syncop_mgmt_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+        synclock_lock (&conf->big_lock);
+        return ret;
+}
+
+int32_t
+_gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_cluster_unlock_rsp rsp         = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        frame = myframe;
+        args  = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        peerinfo->locked = _gf_false;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_collate_errors (args, op_ret, op_errno, NULL,
+                           GLUSTERD_MGMT_CLUSTER_UNLOCK, peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                           int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_unlock_cbk);
+}
+
+
+int
+gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                       uuid_t my_uuid, uuid_t recv_uuid)
+{
+        int                         ret     = -1;
+        gd1_mgmt_cluster_unlock_req req     = {{0},};
+        glusterd_conf_t             *conf   = THIS->private;
+
+        uuid_copy (req.uuid, my_uuid);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_CLUSTER_UNLOCK,
+                                        gd_syncop_mgmt_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+        synclock_lock (&conf->big_lock);
+        return ret;
+}
+
+int32_t
+_gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        int                         ret         = -1;
+        gd1_mgmt_stage_op_rsp       rsp         = {{0},};
+        struct syncargs             *args       = NULL;
+        xlator_t                    *this       = NULL;
+        dict_t                      *rsp_dict   = NULL;
+        call_frame_t                *frame      = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        this  = THIS;
+        frame = myframe;
+        args  = frame->local;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Staging response "
+                        "for 'Volume %s' received from unknown "
+                        "peer: %s", gd_op_list[rsp.op],
+                        uuid_utoa (rsp.uuid));
+                goto out;
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        if (rsp.op == GD_OP_REPLACE_BRICK || rsp.op == GD_OP_QUOTA) {
+                pthread_mutex_lock (&args->lock_dict);
+                {
+                        ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                             rsp_dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "%s",
+                                        "Failed to aggregate response from "
+                                        " node/brick");
+                }
+                pthread_mutex_unlock (&args->lock_dict);
+        }
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
+                           GLUSTERD_MGMT_STAGE_OP, peerinfo, rsp.uuid);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_stage_op_cbk);
+}
+
+
+int
+gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *args,
+                         uuid_t my_uuid, uuid_t recv_uuid, int op,
+                         dict_t *dict_out, dict_t *op_ctx)
+{
+        gd1_mgmt_stage_op_req *req  = NULL;
+        glusterd_conf_t       *conf = THIS->private;
+        int                   ret   = -1;
+
+        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_stage_req_t);
+        if (!req)
+                goto out;
+
+        uuid_copy (req->uuid, my_uuid);
+        req->op = op;
+
+        ret = dict_allocate_and_serialize (dict_out,
+                                           &req->buf.buf_val, &req->buf.buf_len);
+        if (ret)
+                goto out;
+
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (rpc, req, args, NULL, &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_STAGE_OP,
+                                        gd_syncop_stage_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_stage_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gd_stage_op_req_free (req);
+        return ret;
+
+}
+
+int32_t
+_gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        struct syncargs        *args  = NULL;
+        gd1_mgmt_brick_op_rsp  rsp   = {0,};
+        int                    ret   = -1;
+        call_frame_t           *frame = NULL;
+
+        frame = myframe;
+        args = frame->local;
+        frame->local = NULL;
+
+        /* initialize */
+        args->op_ret   = -1;
+        args->op_errno = EINVAL;
+
+        if (-1 == req->rpc_status) {
+                args->op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.output.output_len) {
+                args->dict  = dict_new ();
+                if (!args->dict) {
+                        ret = -1;
+                        args->op_errno = ENOMEM;
+                        goto out;
+                }
+
+                ret = dict_unserialize (rsp.output.output_val,
+                                        rsp.output.output_len,
+                                        &args->dict);
+                if (ret < 0)
+                        goto out;
+        }
+
+        args->op_ret = rsp.op_ret;
+        args->op_errno = rsp.op_errno;
+        args->errstr = gf_strdup (rsp.op_errstr);
+
+out:
+        if ((rsp.op_errstr) && (strcmp (rsp.op_errstr, "") != 0))
+                free (rsp.op_errstr);
+        free (rsp.output.output_val);
+
+        STACK_DESTROY (frame->root);
+        __wake (args);
+
+        return 0;
+}
+
+int32_t
+gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_brick_op_cbk);
+}
+
+int
+gd_syncop_mgmt_brick_op (struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
+                         int op, dict_t *dict_out, dict_t *op_ctx,
+                         char **errstr)
+{
+        struct syncargs        args = {0, };
+        gd1_mgmt_brick_op_req  *req  = NULL;
+        int                    ret  = 0;
+        xlator_t               *this = NULL;
+
+        this = THIS;
+        args.op_ret = -1;
+        args.op_errno = ENOTCONN;
+
+        if ((pnode->type == GD_NODE_NFS) ||
+            (pnode->type == GD_NODE_QUOTAD) ||
+            ((pnode->type == GD_NODE_SHD) && (op == GD_OP_STATUS_VOLUME))) {
+                ret = glusterd_node_op_build_payload
+                        (op, &req, dict_out);
+
+        } else {
+                ret = glusterd_brick_op_build_payload
+                        (op, pnode->node, &req, dict_out);
+
+        }
+
+        if (ret)
+                goto out;
+
+        GD_SYNCOP (rpc, (&args), NULL, gd_syncop_brick_op_cbk, req,
+                   &gd_brick_prog, req->op, xdr_gd1_mgmt_brick_op_req);
+
+        if (args.errstr && errstr)
+                *errstr = args.errstr;
+        else
+                GF_FREE (args.errstr);
+
+        if (GD_OP_STATUS_VOLUME == op) {
+                ret = dict_set_int32 (args.dict, "index", pnode->index);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting index on brick status"
+                                " rsp dict");
+                        args.op_ret = -1;
+                        goto out;
+                }
+        }
+        if (args.op_ret == 0)
+                glusterd_handle_node_rsp (dict_out, pnode->node, op,
+                                          args.dict, op_ctx, errstr,
+                                          pnode->type);
+
+out:
+        errno = args.op_errno;
+        if (args.dict)
+                dict_unref (args.dict);
+        gd_brick_op_req_free (req);
+        return args.op_ret;
+
+}
+
+int32_t
+_gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int                          ret           = -1;
+        gd1_mgmt_commit_op_rsp       rsp           = {{0},};
+        struct syncargs             *args          = NULL;
+        xlator_t                    *this          = NULL;
+        dict_t                      *rsp_dict      = NULL;
+        call_frame_t                *frame         = NULL;
+        glusterd_peerinfo_t         *peerinfo      = NULL;
+        int                          op_ret        = -1;
+        int                          op_errno      = -1;
+        int                          type          = GF_QUOTA_OPTION_TYPE_NONE;
+
+        this  = THIS;
+        frame = myframe;
+        args  = frame->local;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+        if (ret < 0) {
+                goto out;
+        }
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Commit response "
+                        "for 'Volume %s' received from unknown "
+                        "peer: %s", gd_op_list[rsp.op],
+                        uuid_utoa (rsp.uuid));
+                goto out;
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        if (rsp.op == GD_OP_QUOTA) {
+                ret = dict_get_int32 (args->dict, "type", &type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "opcode");
+                        goto out;
+                }
+        }
+
+        if ((rsp.op != GD_OP_QUOTA) || (type == GF_QUOTA_OPTION_TYPE_LIST)) {
+                pthread_mutex_lock (&args->lock_dict);
+                {
+                        ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                             rsp_dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "%s",
+                                        "Failed to aggregate response from "
+                                        " node/brick");
+                }
+                pthread_mutex_unlock (&args->lock_dict);
+        }
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
+                           GLUSTERD_MGMT_COMMIT_OP, peerinfo, rsp.uuid);
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+
+        return 0;
+}
+
+int32_t
+gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_commit_op_cbk);
+}
+
+
+int
+gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *args,
+                          uuid_t my_uuid, uuid_t recv_uuid,
+                          int op, dict_t *dict_out, dict_t *op_ctx)
+{
+        glusterd_conf_t        *conf = THIS->private;
+        gd1_mgmt_commit_op_req *req  = NULL;
+        int                    ret  = -1;
+
+        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_commit_req_t);
+        if (!req)
+                goto out;
+
+        uuid_copy (req->uuid, my_uuid);
+        req->op = op;
+
+        ret = dict_allocate_and_serialize (dict_out,
+                                           &req->buf.buf_val, &req->buf.buf_len);
+        if (ret)
+                goto out;
+
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (rpc, req, args, NULL, &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_COMMIT_OP ,
+                                        gd_syncop_commit_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_commit_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gd_commit_op_req_free (req);
+        return ret;
+}
+
+
+int
+gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers,
+                     glusterd_op_t op)
+{
+        glusterd_peerinfo_t *peerinfo = NULL;
+        int                 npeers      = 0;
+
+        list_for_each_entry (peerinfo, peers, uuid_list) {
+                if (!peerinfo->connected)
+                        continue;
+                if (op != GD_OP_SYNC_VOLUME &&
+                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+                        continue;
+
+                list_add_tail (&peerinfo->op_peers_list, xact_peers);
+                npeers++;
+        }
+        return npeers;
+}
+
+int
+gd_lock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, dict_t *op_ctx,
+                  char **op_errstr, int npeers, uuid_t txn_id)
+{
+        int                 ret         = -1;
+        int                 peer_cnt    = 0;
+        uuid_t              peer_uuid   = {0};
+        xlator_t            *this       = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs     args        = {0};
+        struct list_head   *peers       = NULL;
+
+        peers = &conf->xaction_peers;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        this = THIS;
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                if (conf->op_version < GD_OP_VERSION_4) {
+                        /* Reset lock status */
+                        peerinfo->locked = _gf_false;
+                        gd_syncop_mgmt_lock (peerinfo, &args,
+                                             MY_UUID, peer_uuid);
+                } else
+                        gd_syncop_mgmt_volume_lock (op, op_ctx, peerinfo, &args,
+                                                    MY_UUID, peer_uuid, txn_id);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+                else {
+                        ret = gf_asprintf (op_errstr, "Another transaction "
+                                           "could be in progress. Please try "
+                                           "again after sometime.");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock");
+
+                }
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for 'Volume %s' "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int
+gd_stage_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
+                   dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int                 ret             = -1;
+        int                 peer_cnt           = 0;
+        dict_t              *rsp_dict       = NULL;
+        char                *hostname       = NULL;
+        xlator_t            *this           = NULL;
+        glusterd_peerinfo_t *peerinfo       = NULL;
+        uuid_t              tmp_uuid        = {0};
+        char                *errstr         = NULL;
+        struct syncargs     args            = {0};
+
+        this = THIS;
+        rsp_dict = dict_new ();
+        if (!rsp_dict)
+                goto out;
+
+        ret = glusterd_op_stage_validate (op, req_dict, op_errstr, rsp_dict);
+        if (ret) {
+                hostname = "localhost";
+                goto stage_done;
+        }
+
+        if ((op == GD_OP_REPLACE_BRICK || op == GD_OP_QUOTA)) {
+                ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                "Failed to aggregate response from node/brick");
+                        goto out;
+                }
+        }
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+stage_done:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
+                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
+                        (*op_errstr) ? *op_errstr : " ");
+                if (*op_errstr == NULL)
+                        gf_asprintf (op_errstr, OPERRSTR_STAGE_FAIL, hostname);
+                goto out;
+        }
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                ret = gd_syncop_mgmt_stage_op (peerinfo->rpc, &args,
+                                               MY_UUID, tmp_uuid,
+                                               op, req_dict, op_ctx);
+                peer_cnt++;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent stage op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], peer_cnt);
+
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.errstr)
+                 *op_errstr = gf_strdup (args.errstr);
+        else if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
+                *op_errstr = gf_strdup (errstr);
+
+        ret = args.op_ret;
+
+out:
+        if ((ret == 0) && (op == GD_OP_QUOTA)) {
+                ret = glusterd_validate_and_set_gfid (op_ctx, req_dict,
+                                                      op_errstr);
+                if (ret)
+                        goto out;
+        }
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+        return ret;
+}
+
+int
+gd_commit_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
+                    dict_t *req_dict, char **op_errstr, int npeers)
+{
+        dict_t              *rsp_dict       = NULL;
+        int                 peer_cnt           = -1;
+        int                 ret             = -1;
+        char                *hostname       = NULL;
+        glusterd_peerinfo_t *peerinfo       = NULL;
+        xlator_t            *this           = NULL;
+        uuid_t              tmp_uuid        = {0};
+        char                *errstr         = NULL;
+        struct syncargs     args            = {0};
+        int                 type            = GF_QUOTA_OPTION_TYPE_NONE;
+
+        this = THIS;
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_op_commit_perform (op, req_dict, op_errstr, rsp_dict);
+        if (ret) {
+                hostname = "localhost";
+                goto commit_done;
+        }
+
+        if (op == GD_OP_QUOTA) {
+                ret = dict_get_int32 (op_ctx, "type", &type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "opcode");
+                        goto out;
+                }
+        }
+
+        if (((op == GD_OP_QUOTA) && (type == GF_QUOTA_OPTION_TYPE_LIST)) ||
+            ((op != GD_OP_SYNC_VOLUME) && (op != GD_OP_QUOTA))) {
+
+                ret =  glusterd_syncop_aggr_rsp_dict (op, op_ctx,
+                                                      rsp_dict);
+                if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", "Failed to aggregate "
+                        "response from node/brick");
+                goto out;
+                }
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+commit_done:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
+                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
+                        (*op_errstr) ? *op_errstr : " ");
+                if (*op_errstr == NULL)
+                        gf_asprintf (op_errstr, OPERRSTR_COMMIT_FAIL,
+                                     hostname);
+                goto out;
+        }
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                ret = gd_syncop_mgmt_commit_op (peerinfo->rpc, &args,
+                                                MY_UUID, tmp_uuid,
+                                                op, req_dict, op_ctx);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+        ret = args.op_ret;
+        if (args.errstr)
+                 *op_errstr = gf_strdup (args.errstr);
+        else if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
+                *op_errstr = gf_strdup (errstr);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], peer_cnt);
+out:
+        if (!ret)
+                glusterd_op_modify_op_ctx (op, op_ctx);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        GF_FREE (args.errstr);
+        args.errstr = NULL;
+
+        return ret;
+}
+
+int
+gd_unlock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, int *op_ret,
+                    rpcsvc_request_t *req, dict_t *op_ctx, char *op_errstr,
+                    int npeers, char *volname, gf_boolean_t is_acquired,
+                    uuid_t txn_id)
+{
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        glusterd_peerinfo_t *tmp        = NULL;
+        uuid_t              tmp_uuid    = {0};
+        int                 peer_cnt       = 0;
+        int                 ret         = -1;
+        xlator_t            *this       = NULL;
+        struct syncargs     args        = {0};
+        struct list_head   *peers       = NULL;
+
+        peers = &conf->xaction_peers;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* If the lock has not been held during this
+         * transaction, do not send unlock requests */
+        if (!is_acquired) {
+                ret = 0;
+                goto out;
+        }
+
+        this = THIS;
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        if (conf->op_version < GD_OP_VERSION_4) {
+                list_for_each_entry_safe (peerinfo, tmp, peers, op_peers_list) {
+                        /* Only unlock peers that were locked */
+                        if (peerinfo->locked) {
+                                gd_syncop_mgmt_unlock (peerinfo, &args,
+                                                       MY_UUID, tmp_uuid);
+                                peer_cnt++;
+                                list_del_init (&peerinfo->op_peers_list);
+                        }
+                }
+        } else {
+                if (volname) {
+                        list_for_each_entry_safe (peerinfo, tmp,
+                                                  peers, op_peers_list) {
+                                gd_syncop_mgmt_volume_unlock (op_ctx, peerinfo,
+                                                              &args, MY_UUID,
+                                                              tmp_uuid, txn_id);
+                                peer_cnt++;
+                                list_del_init (&peerinfo->op_peers_list);
+                        }
+                }
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent unlock op req for 'Volume %s' "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to unlock "
+                        "on some peer(s)");
+        }
+
+out:
+        /* If unlock failed, and op_ret was previously set
+         * priority is given to the op_ret. If op_ret was
+         * not set, and unlock failed, then set op_ret */
+        if (!*op_ret)
+                *op_ret = ret;
+
+        if (is_acquired) {
+                /* Based on the op-version,
+                 * we release the cluster or volume lock
+                 * and clear the op */
+
+                glusterd_op_clear_op (op);
+                if (conf->op_version < GD_OP_VERSION_4)
+                        glusterd_unlock (MY_UUID);
+                else {
+                        if (volname) {
+                                ret = glusterd_volume_unlock (volname, MY_UUID);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to release lock for %s",
+                                                volname);
+                        }
+                }
+        }
+
+        if (!*op_ret)
+                *op_ret = ret;
+
+        return 0;
+}
+
+int
+gd_get_brick_count (struct list_head *bricks)
+{
+        glusterd_pending_node_t *pending_node = NULL;
+        int                     npeers        = 0;
+        list_for_each_entry (pending_node, bricks, list) {
+                npeers++;
+        }
+        return npeers;
+}
+
+int
+gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr)
+{
+        glusterd_pending_node_t *pending_node = NULL;
+        struct list_head        selected = {0,};
+        xlator_t                *this = NULL;
+        int                     brick_count = 0;
+        int                     ret = -1;
+        rpc_clnt_t              *rpc = NULL;
+        dict_t                  *rsp_dict = NULL;
+        glusterd_conf_t         *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&selected);
+        ret = glusterd_op_bricks_select (op, req_dict, op_errstr, &selected, rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                       (*op_errstr)? *op_errstr: "Brick op failed. Check "
+                       "glusterd log file for more details.");
+                goto out;
+        }
+
+        if (op == GD_OP_HEAL_VOLUME) {
+                ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
+                if (ret)
+                        goto out;
+        }
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        brick_count = 0;
+        list_for_each_entry (pending_node, &selected, list) {
+                rpc = glusterd_pending_node_get_rpc (pending_node);
+                if (!rpc) {
+                        if (pending_node->type == GD_NODE_REBALANCE) {
+                                ret = 0;
+                                glusterd_defrag_volume_node_rsp (req_dict,
+                                                                 NULL, op_ctx);
+                                goto out;
+                        }
+
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
+                                "due to rpc failure.");
+                        goto out;
+                }
+                ret = gd_syncop_mgmt_brick_op (rpc, pending_node, op, req_dict,
+                                               op_ctx, op_errstr);
+                if (ret)
+                        goto out;
+
+                brick_count++;
+        }
+
+        ret = 0;
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+        gf_log (this->name, GF_LOG_DEBUG, "Sent op req to %d bricks",
+                brick_count);
+        return ret;
+}
+
+void
+gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
+{
+        int                         ret              = -1;
+        int                         op_ret           = -1;
+        int                         npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        glusterd_op_t               op               = 0;
+        int32_t                     tmp_op           = 0;
+        char                        *op_errstr       = NULL;
+        char                        *tmp             = NULL;
+        char                        *volname         = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *txn_id          = NULL;
+        glusterd_op_info_t          txn_opinfo;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume "
+                        "operation");
+                goto out;
+        }
+        op = tmp_op;
+
+        /* Generate a transaction-id for this operation and
+         * save it in the dict */
+        ret = glusterd_generate_txn_id (op_ctx, &txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to generate transaction id");
+                goto out;
+        }
+
+        /* Save opinfo for this transaction with the transaction id */
+        glusterd_txn_opinfo_init (&txn_opinfo, NULL, &op, NULL, NULL);
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_opinfo);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Transaction ID : %s", uuid_utoa (*txn_id));
+
+        opinfo = txn_opinfo;
+
+        /* Save the MY_UUID as the originator_uuid */
+        ret = glusterd_set_originator_uuid (op_ctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        /* Based on the op_version, acquire a cluster or volume lock */
+        if (conf->op_version < GD_OP_VERSION_4) {
+                ret = glusterd_lock (MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock");
+                        gf_asprintf (&op_errstr,
+                                     "Another transaction is in progress. "
+                                     "Please try again after sometime.");
+                        goto out;
+                }
+        } else {
+
+                /* If no volname is given as a part of the command, locks will
+                 * not be held */
+                ret = dict_get_str (op_ctx, "volname", &tmp);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Failed to get volume "
+                                "name");
+                        goto local_locking_done;
+                } else {
+                        /* Use a copy of volname, as cli response will be
+                         * sent before the unlock, and the volname in the
+                         * dict, might be removed */
+                        volname = gf_strdup (tmp);
+                        if (!volname)
+                                goto out;
+                }
+
+                ret = glusterd_volume_lock (volname, MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock for %s", volname);
+                        gf_asprintf (&op_errstr,
+                                     "Another transaction is in progress "
+                                     "for %s. Please try again after sometime.",
+                                     volname);
+                        goto out;
+                }
+        }
+
+        is_acquired = _gf_true;
+
+local_locking_done:
+
+        INIT_LIST_HEAD (&conf->xaction_peers);
+
+        npeers = gd_build_peers_list  (&conf->peers, &conf->xaction_peers, op);
+
+        /* If no volname is given as a part of the command, locks will
+         * not be held */
+        if (volname || (conf->op_version < GD_OP_VERSION_4)) {
+                ret = gd_lock_op_phase (conf, op, op_ctx, &op_errstr,
+                                        npeers, *txn_id);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Locking Peers Failed.");
+                        goto out;
+                }
+        }
+
+        ret = glusterd_op_build_payload (&req_dict, &op_errstr, op_ctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        ret = gd_stage_op_phase (&conf->xaction_peers, op, op_ctx, req_dict,
+                                 &op_errstr, npeers);
+        if (ret)
+                goto out;
+
+        ret = gd_brick_op_phase (op, op_ctx, req_dict, &op_errstr);
+        if (ret)
+                goto out;
+
+        ret = gd_commit_op_phase (&conf->xaction_peers, op, op_ctx, req_dict,
+                                  &op_errstr, npeers);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        op_ret = ret;
+        if (txn_id) {
+                (void) gd_unlock_op_phase (conf, op, &op_ret, req,
+                                           op_ctx, op_errstr,
+                                           npeers, volname,
+                                           is_acquired, *txn_id);
+
+                /* Clearing the transaction opinfo */
+                ret = glusterd_clear_txn_opinfo (txn_id);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to clear transaction's "
+                                "opinfo for transaction ID : %s",
+                                uuid_utoa (*txn_id));
+        }
+
+        glusterd_op_send_cli_response (op, op_ret, 0, req, op_ctx, op_errstr);
+
+        if (volname)
+                GF_FREE (volname);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return;
+}
+
+int32_t
+glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
+                            void *dict)
+{
+        int              ret = 0;
+
+        ret = dict_set_int32 (dict, GD_SYNC_OPCODE_KEY, op);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "dict set failed for setting operations");
+                goto out;
+        }
+
+        gd_sync_task_begin (dict, req);
+        ret = 0;
+out:
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h
new file mode 100644
index 000000000..7e4d47f9a
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h
@@ -0,0 +1,54 @@
+/*
+   Copyright (c) 2012-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __RPC_SYNCOP_H
+#define __RPC_SYNCOP_H
+
+#include "syncop.h"
+#include "glusterd-sm.h"
+
+#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
+
+/* gd_syncop_* */
+#define GD_SYNCOP(rpc, stb, cookie, cbk, req, prog, procnum, xdrproc) do {     \
+                int ret = 0;                                                   \
+                struct  synctask        *task = NULL;                          \
+                glusterd_conf_t         *conf= THIS->private;                  \
+                                                                               \
+                task = synctask_get ();                                        \
+                stb->task = task;                                              \
+                                                                               \
+                /*This is to ensure that the brick_op_cbk is able to           \
+                 * take the big lock*/                                         \
+                synclock_unlock (&conf->big_lock);                             \
+                ret = gd_syncop_submit_request (rpc, req, stb, cookie,         \
+                                                prog, procnum, cbk,            \
+                                                (xdrproc_t)xdrproc);           \
+                if (!ret)                                                      \
+                        synctask_yield (stb->task);                            \
+                synclock_lock (&conf->big_lock);                               \
+        } while (0)
+
+
+int gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
+                              void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                              fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+
+
+int gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                         uuid_t my_uuid, uuid_t recv_uuid);
+int gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                           uuid_t my_uuid, uuid_t recv_uuid);
+int gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *arg,
+                             uuid_t my_uuid, uuid_t recv_uuid, int op,
+                             dict_t *dict_out, dict_t *op_ctx);
+int gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *arg,
+                              uuid_t my_uuid, uuid_t recv_uuid, int op,
+                              dict_t *dict_out, dict_t *op_ctx);
+#endif /* __RPC_SYNCOP_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 58a894532..fdfdcc281 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -32,7 +22,7 @@
 #include "timer.h"
 #include "defaults.h"
 #include "compat.h"
-#include "md5.h"
+#include "syncop.h"
 #include "run.h"
 #include "compat-errno.h"
 #include "statedump.h"
@@ -45,9 +35,11 @@
 #include "glusterd-store.h"
 #include "glusterd-volgen.h"
 #include "glusterd-pmap.h"
+#include "glusterfs-acl.h"
+#include "glusterd-syncop.h"
+#include "glusterd-locks.h"
 
 #include "xdr-generic.h"
-
 #include <sys/resource.h>
 #include <inttypes.h>
 #include <signal.h>
@@ -59,6 +51,11 @@
 #include <unistd.h>
 #include <fnmatch.h>
 #include <sys/statvfs.h>
+#include <ifaddrs.h>
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
 
 #ifdef GF_LINUX_HOST_OS
 #include <mntent.h>
@@ -68,15 +65,34 @@
 #include <sys/sockio.h>
 #endif
 
-#define MOUNT_PROGRAM 100005
-#define NFS_PROGRAM 100003
-#define NFSV3_VERSION 3
-#define MOUNTV3_VERSION 3
-#define MOUNTV1_VERSION 1
+#define NFS_PROGRAM         100003
+#define NFSV3_VERSION       3
+
+#define MOUNT_PROGRAM       100005
+#define MOUNTV3_VERSION     3
+#define MOUNTV1_VERSION     1
+
+#define NLM_PROGRAM         100021
+#define NLMV4_VERSION       4
+#define NLMV1_VERSION       1
+
+#define CEILING_POS(X) (((X)-(int)(X)) > 0 ? (int)((X)+1) : (int)(X))
 
-char    *glusterd_sock_dir = "/tmp";
 static glusterd_lock_t lock;
 
+char*
+gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo)
+{
+        if ((peerinfo == NULL) || uuid_is_null (peerinfo->uuid))
+                return NULL;
+
+        if (peerinfo->uuid_str[0] == '\0')
+                uuid_utoa_r (peerinfo->uuid, peerinfo->uuid_str);
+
+        return peerinfo->uuid_str;
+}
+
+
 int32_t
 glusterd_get_lock_owner (uuid_t *uuid)
 {
@@ -114,131 +130,6 @@ glusterd_is_fuse_available ()
                 return _gf_false;
 }
 
-gf_boolean_t
-glusterd_is_loopback_localhost (const struct sockaddr *sa, char *hostname)
-{
-        GF_ASSERT (sa);
-
-        gf_boolean_t is_local = _gf_false;
-        const struct in_addr *addr4 = NULL;
-        const struct in6_addr *addr6 = NULL;
-        uint8_t      *ap   = NULL;
-        struct in6_addr loopbackaddr6 = IN6ADDR_LOOPBACK_INIT;
-
-        switch (sa->sa_family) {
-                case AF_INET:
-                        addr4 = &(((struct sockaddr_in *)sa)->sin_addr);
-                        ap = (uint8_t*)&addr4->s_addr;
-                        if (ap[0] == 127)
-                                is_local = _gf_true;
-                        break;
-
-                case AF_INET6:
-                        addr6 = &(((struct sockaddr_in6 *)sa)->sin6_addr);
-                        if (memcmp (addr6, &loopbackaddr6,
-                                    sizeof (loopbackaddr6)) == 0)
-                                is_local = _gf_true;
-                        break;
-
-                default:
-                        if (hostname)
-                                gf_log ("glusterd", GF_LOG_ERROR,
-                                        "unknown address family %d for %s",
-                                        sa->sa_family, hostname);
-                        break;
-        }
-
-        return is_local;
-}
-
-char *
-get_ip_from_addrinfo (struct addrinfo *addr, char **ip)
-{
-        char buf[64];
-        void *in_addr = NULL;
-        struct sockaddr_in *s4 = NULL;
-        struct sockaddr_in6 *s6 = NULL;
-
-        switch (addr->ai_family)
-        {
-                case AF_INET:
-                        s4 = (struct sockaddr_in *)addr->ai_addr;
-                        in_addr = &s4->sin_addr;
-                        break;
-
-                case AF_INET6:
-                        s6 = (struct sockaddr_in6 *)addr->ai_addr;
-                        in_addr = &s6->sin6_addr;
-                        break;
-
-                default:
-                        gf_log ("glusterd", GF_LOG_ERROR, "Invalid family");
-                        return NULL;
-        }
-
-        if (!inet_ntop(addr->ai_family, in_addr, buf, sizeof(buf))) {
-                gf_log ("glusterd", GF_LOG_ERROR, "String conversion failed");
-                return NULL;
-        }
-
-        *ip = strdup (buf);
-        return *ip;
-}
-
-int32_t
-glusterd_is_local_addr (char *hostname)
-{
-        int32_t         ret = -1;
-        struct          addrinfo *result = NULL;
-        struct          addrinfo *res = NULL;
-        int32_t         found = 0;
-        int             sd = -1;
-        char            *ip = NULL;
-        xlator_t        *this = NULL;
-
-        this = THIS;
-        ret = getaddrinfo (hostname, NULL, NULL, &result);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "error in getaddrinfo: %s\n",
-                        gai_strerror(ret));
-                goto out;
-        }
-
-        for (res = result; res != NULL; res = res->ai_next) {
-                found = glusterd_is_loopback_localhost (res->ai_addr, hostname);
-                if (found)
-                        goto out;
-        }
-
-        for (res = result; res != NULL; res = res->ai_next) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s ",
-                        get_ip_from_addrinfo (res, &ip));
-                sd = socket (res->ai_family, SOCK_DGRAM, 0);
-                if (sd == -1)
-                        goto out;
-                /*If bind succeeds then its a local address*/
-                ret = bind (sd, res->ai_addr, res->ai_addrlen);
-                if (ret == 0) {
-                        found = _gf_true;
-                        gf_log (this->name, GF_LOG_DEBUG, "%s is local",
-                                get_ip_from_addrinfo (res, &ip));
-                        close (sd);
-                        break;
-                }
-                close (sd);
-        }
-
-out:
-        if (result)
-                freeaddrinfo (result);
-
-        if (!found)
-                gf_log (this->name, GF_LOG_DEBUG, "%s is not local", hostname);
-
-        return !found;
-}
-
 int32_t
 glusterd_lock (uuid_t   uuid)
 {
@@ -247,13 +138,17 @@ glusterd_lock (uuid_t   uuid)
         char    new_owner_str[50];
         char    owner_str[50];
         int     ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (uuid);
 
         glusterd_get_lock_owner (&owner);
 
         if (!uuid_is_null (owner)) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get lock"
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get lock"
                         " for uuid: %s, lock held by: %s",
                         uuid_utoa_r (uuid, new_owner_str),
                         uuid_utoa_r (owner, owner_str));
@@ -263,7 +158,7 @@ glusterd_lock (uuid_t   uuid)
         ret = glusterd_set_lock_owner (uuid);
 
         if (!ret) {
-                gf_log ("glusterd", GF_LOG_INFO, "Cluster lock held by"
+                gf_log (this->name, GF_LOG_DEBUG, "Cluster lock held by"
                          " %s", uuid_utoa (uuid));
         }
 
@@ -279,21 +174,25 @@ glusterd_unlock (uuid_t uuid)
         char    new_owner_str[50];
         char    owner_str[50];
         int32_t ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (uuid);
 
         glusterd_get_lock_owner (&owner);
 
-        if (NULL == owner) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Cluster lock not held!");
+        if (uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR, "Cluster lock not held!");
                 goto out;
         }
 
         ret = uuid_compare (uuid, owner);
 
         if (ret) {
-               gf_log ("glusterd", GF_LOG_ERROR, "Cluster lock held by %s"
-                        " ,unlock req from %s!", uuid_utoa_r (owner ,owner_str)
+               gf_log (this->name, GF_LOG_ERROR, "Cluster lock held by %s ,"
+                       "unlock req from %s!", uuid_utoa_r (owner ,owner_str)
                         , uuid_utoa_r (uuid, new_owner_str));
                goto out;
         }
@@ -301,7 +200,7 @@ glusterd_unlock (uuid_t uuid)
         ret = glusterd_unset_lock_owner (uuid);
 
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to clear cluster "
+                gf_log (this->name, GF_LOG_ERROR, "Unable to clear cluster "
                         "lock");
                 goto out;
         }
@@ -322,16 +221,17 @@ glusterd_get_uuid (uuid_t *uuid)
 
         GF_ASSERT (priv);
 
-        uuid_copy (*uuid, priv->uuid);
+        uuid_copy (*uuid, MY_UUID);
 
         return 0;
 }
 
 int
-glusterd_submit_request (struct rpc_clnt *rpc, void *req,
-                         call_frame_t *frame, rpc_clnt_prog_t *prog,
-                         int procnum, struct iobref *iobref,
-                         xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+glusterd_submit_request_unlocked (struct rpc_clnt *rpc, void *req,
+				  call_frame_t *frame, rpc_clnt_prog_t *prog,
+				  int procnum, struct iobref *iobref,
+				  xlator_t *this, fop_cbk_fn_t cbkfn,
+				  xdrproc_t xdrproc)
 {
         int                     ret         = -1;
         struct iobuf            *iobuf      = NULL;
@@ -402,6 +302,28 @@ out:
         return ret;
 }
 
+
+int
+glusterd_submit_request (struct rpc_clnt *rpc, void *req,
+                         call_frame_t *frame, rpc_clnt_prog_t *prog,
+                         int procnum, struct iobref *iobref,
+                         xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+        glusterd_conf_t         *priv = THIS->private;
+	int ret = -1;
+
+	synclock_unlock (&priv->big_lock);
+	{
+		ret = glusterd_submit_request_unlocked (rpc, req, frame, prog,
+							procnum, iobref, this,
+							cbkfn, xdrproc);
+	}
+	synclock_lock (&priv->big_lock);
+
+	return ret;
+}
+
+
 struct iobuf *
 glusterd_serialize_reply (rpcsvc_request_t *req, void *arg,
                           struct iovec *outmsg, xdrproc_t xdrproc)
@@ -515,7 +437,7 @@ glusterd_check_volume_exists (char *volname)
         ret = stat (pathname, &stbuf);
 
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Volume %s does not exist."
+                gf_log (THIS->name, GF_LOG_DEBUG, "Volume %s does not exist."
                         "stat failed with errno : %d on path: %s",
                         volname, errno, pathname);
                 return _gf_false;
@@ -524,6 +446,37 @@ glusterd_check_volume_exists (char *volname)
         return _gf_true;
 }
 
+glusterd_volinfo_t *
+glusterd_volinfo_unref (glusterd_volinfo_t *volinfo)
+{
+        int refcnt = -1;
+
+        pthread_mutex_lock (&volinfo->reflock);
+        {
+                refcnt = --volinfo->refcnt;
+        }
+        pthread_mutex_unlock (&volinfo->reflock);
+
+        if (!refcnt) {
+                glusterd_volinfo_delete (volinfo);
+                return NULL;
+        }
+
+        return volinfo;
+}
+
+glusterd_volinfo_t *
+glusterd_volinfo_ref (glusterd_volinfo_t *volinfo)
+{
+        pthread_mutex_lock (&volinfo->reflock);
+        {
+                ++volinfo->refcnt;
+        }
+        pthread_mutex_unlock (&volinfo->reflock);
+
+        return volinfo;
+}
+
 int32_t
 glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
 {
@@ -543,28 +496,27 @@ glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
 
         new_volinfo->dict = dict_new ();
         if (!new_volinfo->dict) {
-                if (new_volinfo)
-                        GF_FREE (new_volinfo);
+                GF_FREE (new_volinfo);
 
                 goto out;
         }
 
         new_volinfo->gsync_slaves = dict_new ();
         if (!new_volinfo->gsync_slaves) {
-                if (new_volinfo)
-                        GF_FREE (new_volinfo);
+                GF_FREE (new_volinfo);
 
                 goto out;
         }
 
         new_volinfo->xl = THIS;
 
-        *volinfo = new_volinfo;
+        pthread_mutex_init (&new_volinfo->reflock, NULL);
+        *volinfo = glusterd_volinfo_ref (new_volinfo);
 
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -573,11 +525,9 @@ glusterd_auth_cleanup (glusterd_volinfo_t *volinfo) {
 
         GF_ASSERT (volinfo);
 
-        if (volinfo->auth.username)
-                GF_FREE (volinfo->auth.username);
+        GF_FREE (volinfo->auth.username);
 
-        if (volinfo->auth.password)
-                GF_FREE (volinfo->auth.password);
+        GF_FREE (volinfo->auth.password);
 }
 
 char *
@@ -625,8 +575,7 @@ glusterd_brickinfo_delete (glusterd_brickinfo_t *brickinfo)
 
         list_del_init (&brickinfo->brick_list);
 
-        if (brickinfo->logfile)
-                GF_FREE (brickinfo->logfile);
+        GF_FREE (brickinfo->logfile);
         GF_FREE (brickinfo);
 
         ret = 0;
@@ -651,10 +600,18 @@ glusterd_volume_brickinfos_delete (glusterd_volinfo_t *volinfo)
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+int
+glusterd_volinfo_remove (glusterd_volinfo_t *volinfo)
+{
+        list_del_init (&volinfo->vol_list);
+        glusterd_volinfo_unref (volinfo);
+        return 0;
+}
+
 int32_t
 glusterd_volinfo_delete (glusterd_volinfo_t *volinfo)
 {
@@ -671,16 +628,20 @@ glusterd_volinfo_delete (glusterd_volinfo_t *volinfo)
                 dict_unref (volinfo->dict);
         if (volinfo->gsync_slaves)
                 dict_unref (volinfo->gsync_slaves);
-        if (volinfo->logdir)
-                GF_FREE (volinfo->logdir);
+        GF_FREE (volinfo->logdir);
+        if (volinfo->rebal.dict)
+                dict_unref (volinfo->rebal.dict);
+
+        gf_store_handle_destroy (volinfo->quota_conf_shandle);
 
         glusterd_auth_cleanup (volinfo);
 
+        pthread_mutex_destroy (&volinfo->reflock);
         GF_FREE (volinfo);
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -706,25 +667,53 @@ glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo)
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+int
+glusterd_get_next_available_brickid (glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t *brickinfo    = NULL;
+        char                 *token        = NULL;
+        int                   brickid      = 0;
+        int                   max_brickid  = -1;
+        int                   ret          = -1;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                token = strrchr (brickinfo->brick_id, '-');
+                ret = gf_string2int32 (++token, &brickid);
+                if (ret < 0) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Unable to generate brick ID");
+                        return ret;
+                }
+                if (brickid > max_brickid)
+                        max_brickid = brickid;
+        }
+
+        return max_brickid + 1 ;
+}
+
 int32_t
 glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo)
 {
         int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (brickinfo);
 
         ret = glusterd_hostname_to_uuid (brickinfo->hostname, brickinfo->uuid);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
-glusterd_brickinfo_from_brick (char *brick,
-                               glusterd_brickinfo_t **brickinfo)
+glusterd_brickinfo_new_from_brick (char *brick,
+                                   glusterd_brickinfo_t **brickinfo)
 {
         int32_t                 ret = -1;
         glusterd_brickinfo_t    *new_brickinfo = NULL;
@@ -732,6 +721,7 @@ glusterd_brickinfo_from_brick (char *brick,
         char                    *path = NULL;
         char                    *tmp_host = NULL;
         char                    *tmp_path = NULL;
+        char                    *vg       = NULL;
 
         GF_ASSERT (brick);
         GF_ASSERT (brickinfo);
@@ -747,7 +737,21 @@ glusterd_brickinfo_from_brick (char *brick,
         GF_ASSERT (path);
 
         ret = glusterd_brickinfo_new (&new_brickinfo);
+        if (ret)
+                goto out;
 
+#ifdef HAVE_BD_XLATOR
+        vg = strchr (path, '?');
+        /* ? is used as a delimiter for vg */
+        if (vg) {
+                strncpy (new_brickinfo->vg, vg + 1, PATH_MAX - 1);
+                *vg = '\0';
+        }
+        new_brickinfo->caps = CAPS_BD;
+#else
+        vg = NULL; /* Avoid compiler warnings when BD not enabled */
+#endif
+        ret = gf_canonicalize_path (path);
         if (ret)
                 goto out;
 
@@ -758,23 +762,272 @@ glusterd_brickinfo_from_brick (char *brick,
 
         ret = 0;
 out:
-        if (tmp_host)
-                GF_FREE (tmp_host);
+        GF_FREE (tmp_host);
         if (tmp_host)
                 GF_FREE (tmp_path);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static gf_boolean_t
+_is_prefix (char *str1, char *str2)
+{
+        GF_ASSERT (str1);
+        GF_ASSERT (str2);
+
+        int             i = 0;
+        int             len1 = 0;
+        int             len2 = 0;
+        int             small_len = 0;
+        char            *bigger = NULL;
+        gf_boolean_t    prefix = _gf_true;
+
+        len1 = strlen (str1);
+        len2 = strlen (str2);
+        small_len = min (len1, len2);
+        for (i = 0; i < small_len; i++) {
+                if (str1[i] != str2[i]) {
+                        prefix = _gf_false;
+                        break;
+                }
+        }
+
+        if (len1 < len2)
+            bigger = str2;
+
+        else if (len1 > len2)
+            bigger = str1;
+
+        else
+            return prefix;
+
+        if (bigger[small_len] != '/')
+            prefix = _gf_false;
+
+        return prefix;
+}
+
+/* Checks if @path is available in the peer identified by @uuid
+ * 'availability' is determined by querying current state of volumes
+ * in the cluster. */
+gf_boolean_t
+glusterd_is_brickpath_available (uuid_t uuid, char *path)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
+        glusterd_conf_t         *priv      = NULL;
+        gf_boolean_t            available  = _gf_false;
+        char                    tmp_path[PATH_MAX+1] = {0};
+        char                    tmp_brickpath[PATH_MAX+1] = {0};
+
+        priv = THIS->private;
+
+        strncpy (tmp_path, path, PATH_MAX);
+        /* path may not yet exist */
+        if (!realpath (path, tmp_path)) {
+                if (errno != ENOENT) {
+                        goto out;
+                }
+                /* When realpath(3) fails, tmp_path is undefined. */
+                strncpy(tmp_path,path,PATH_MAX);
+        }
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        if (uuid_compare (uuid, brickinfo->uuid))
+                                continue;
+
+                        if (!realpath (brickinfo->path, tmp_brickpath)) {
+                            if (errno == ENOENT)
+                                strncpy (tmp_brickpath, brickinfo->path,
+                                         PATH_MAX);
+                            else
+                                goto out;
+                        }
+
+                        if (_is_prefix (tmp_brickpath, tmp_path))
+                                goto out;
+                }
+        }
+        available = _gf_true;
+out:
+        return available;
+}
+
+#ifdef HAVE_BD_XLATOR
+/*
+ * Sets the tag of the format "trusted.glusterfs.volume-id:<uuid>" in
+ * the brick VG. It is used to avoid using same VG for another brick.
+ * @volume-id - gfid, @brick - brick info, @msg - Error message returned
+ * to the caller
+ */
+int
+glusterd_bd_set_vg_tag (unsigned char *volume_id, glusterd_brickinfo_t *brick,
+                        char *msg, int msg_size)
+{
+        lvm_t        handle    = NULL;
+        vg_t         vg        = NULL;
+        char        *uuid      = NULL;
+        int          ret       = -1;
+
+        gf_asprintf (&uuid, "%s:%s", GF_XATTR_VOL_ID_KEY,
+                     uuid_utoa (volume_id));
+        if (!uuid) {
+                snprintf (msg, sizeof(*msg), "Could not allocate memory "
+                          "for tag");
+                return -1;
+        }
+
+        handle = lvm_init (NULL);
+        if (!handle) {
+                snprintf (msg, sizeof(*msg), "lvm_init failed");
+                goto out;
+        }
+
+        vg = lvm_vg_open (handle, brick->vg, "w", 0);
+        if (!vg) {
+                snprintf (msg, sizeof(*msg), "Could not open VG %s",
+                          brick->vg);
+                goto out;
+        }
+
+        if (lvm_vg_add_tag (vg, uuid) < 0) {
+                snprintf (msg, sizeof(*msg), "Could not set tag %s for "
+                          "VG %s", uuid, brick->vg);
+                goto out;
+        }
+        lvm_vg_write (vg);
+        ret = 0;
+out:
+        GF_FREE (uuid);
+
+        if (vg)
+                lvm_vg_close (vg);
+        if (handle)
+                lvm_quit (handle);
+
+        return ret;
+}
+#endif
+
+int
+glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
+                                        uuid_t volume_id, char **op_errstr,
+                                        gf_boolean_t is_force)
+{
+        int          ret                 = -1;
+        char         parentdir[PATH_MAX] = {0,};
+        struct stat  parent_st           = {0,};
+        struct stat  brick_st            = {0,};
+        struct stat  root_st             = {0,};
+        char         msg[2048]           = {0,};
+        gf_boolean_t is_created          = _gf_false;
+
+        ret = mkdir (brickinfo->path, 0777);
+        if (ret) {
+                if (errno != EEXIST) {
+                        snprintf (msg, sizeof (msg), "Failed to create brick "
+                                  "directory for brick %s:%s. Reason : %s ",
+                                  brickinfo->hostname, brickinfo->path,
+                                  strerror (errno));
+                        goto out;
+                }
+        } else {
+                is_created = _gf_true;
+        }
+
+        ret = lstat (brickinfo->path, &brick_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on %s. Reason : %s",
+                          brickinfo->path, strerror (errno));
+                goto out;
+        }
+
+        if ((!is_created) && (!S_ISDIR (brick_st.st_mode))) {
+                snprintf (msg, sizeof (msg), "The provided path %s which is "
+                          "already present, is not a directory",
+                          brickinfo->path);
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (parentdir, sizeof (parentdir), "%s/..", brickinfo->path);
+
+        ret = lstat ("/", &root_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on /. Reason : %s",
+                          strerror (errno));
+                goto out;
+        }
+
+        ret = lstat (parentdir, &parent_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on %s. Reason : %s",
+                          parentdir, strerror (errno));
+                goto out;
+        }
+
+        if (!is_force) {
+                if (brick_st.st_dev != parent_st.st_dev) {
+                        snprintf (msg, sizeof (msg), "The brick %s:%s is a "
+                                  "mount point. Please create a sub-directory "
+                                  "under the mount point and use that as the "
+                                  "brick directory. Or use 'force' at the end "
+                                  "of the command if you want to override this "
+                                  "behavior.", brickinfo->hostname,
+                                  brickinfo->path);
+                        ret = -1;
+                        goto out;
+                }
+                else if (parent_st.st_dev == root_st.st_dev) {
+                        snprintf (msg, sizeof (msg), "The brick %s:%s "
+                                  "is being created in the root partition. It "
+                                  "is recommended that you don't use the "
+                                  "system's root partition for storage backend."
+                                  " Or use 'force' at the end of the command if"
+                                  " you want to override this behavior.",
+                                  brickinfo->hostname, brickinfo->path);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+#ifdef HAVE_BD_XLATOR
+        if (brickinfo->vg[0]) {
+                ret = glusterd_bd_set_vg_tag (volume_id, brickinfo, msg,
+                                              sizeof(msg));
+                if (ret)
+                        goto out;
+        }
+#endif
+        ret = glusterd_check_and_set_brick_xattr (brickinfo->hostname,
+                                                  brickinfo->path, volume_id,
+                                                  op_errstr, is_force);
+        if (ret)
+                goto out;
+
+        ret = 0;
+
+out:
+        if (ret && is_created)
+                rmdir (brickinfo->path);
+        if (ret && !*op_errstr && msg[0] != '\0')
+                *op_errstr = gf_strdup (msg);
+
         return ret;
 }
 
 int32_t
 glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
                                glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t **brickinfo,
-                               gf_path_match_t path_match)
+                               glusterd_brickinfo_t **brickinfo)
 {
         glusterd_brickinfo_t    *brickiter = NULL;
         uuid_t                  peer_uuid = {0};
         int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
 
         if (uuid) {
                 uuid_copy (peer_uuid, uuid);
@@ -786,80 +1039,48 @@ glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
         ret = -1;
         list_for_each_entry (brickiter, &volinfo->bricks, brick_list) {
 
-                if (uuid_is_null (brickiter->uuid) &&
-                    glusterd_resolve_brick (brickiter))
+                if ((uuid_is_null (brickiter->uuid)) &&
+                    (glusterd_resolve_brick (brickiter) != 0))
                         goto out;
                 if (uuid_compare (peer_uuid, brickiter->uuid))
                         continue;
 
-                if (!strcmp (brickiter->path, path)) {
-                        gf_log (THIS->name, GF_LOG_INFO, "Found brick");
+                if (strcmp (brickiter->path, path) == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG, LOGSTR_FOUND_BRICK,
+                                brickiter->hostname, brickiter->path,
+                                volinfo->volname);
                         ret = 0;
                         if (brickinfo)
                                 *brickinfo = brickiter;
                         break;
                 }
-
-                if (path_match != GF_PATH_PARTIAL)
-                        continue;
-
-#ifdef GF_LINUX_HOST_OS
-                if (!fnmatch (path, brickiter->path, FNM_LEADING_DIR) ||
-                    !fnmatch (brickiter->path, path, FNM_LEADING_DIR)) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "paths %s and %s are recursive",
-                                path, brickiter->path);
-                        *brickinfo = brickiter;
-                        ret = 0;
-                        break;
-                }
-#endif
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int32_t
 glusterd_volume_brickinfo_get_by_brick (char *brick,
                                         glusterd_volinfo_t *volinfo,
-                                        glusterd_brickinfo_t **brickinfo,
-                                        gf_path_match_t path_match)
+                                        glusterd_brickinfo_t **brickinfo)
 {
         int32_t                 ret = -1;
-        char                    *hostname = NULL;
-        char                    *path = NULL;
-        char                    *tmp_host = NULL;
-        char                    *tmp_path = NULL;
+        glusterd_brickinfo_t    *tmp_brickinfo = NULL;
 
         GF_ASSERT (brick);
         GF_ASSERT (volinfo);
 
-        gf_log ("", GF_LOG_INFO, "brick: %s", brick);
-
-        tmp_host = gf_strdup (brick);
-        if (tmp_host)
-                get_host_name (tmp_host, &hostname);
-        tmp_path = gf_strdup (brick);
-        if (tmp_path)
-                get_path_name (tmp_path, &path);
-
-        if (!hostname || !path) {
-                gf_log ("", GF_LOG_ERROR,
-                        "brick %s is not of form <HOSTNAME>:<export-dir>",
-                        brick);
-                ret = -1;
+        ret = glusterd_brickinfo_new_from_brick (brick, &tmp_brickinfo);
+        if (ret)
                 goto out;
-        }
 
-        ret = glusterd_volume_brickinfo_get (NULL, hostname, path, volinfo,
-                                             brickinfo, path_match);
+        ret = glusterd_volume_brickinfo_get (NULL, tmp_brickinfo->hostname,
+                                             tmp_brickinfo->path, volinfo,
+                                             brickinfo);
+        (void) glusterd_brickinfo_delete (tmp_brickinfo);
 out:
-        if (tmp_host)
-                GF_FREE (tmp_host);
-        if (tmp_path)
-                GF_FREE (tmp_path);
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -873,7 +1094,7 @@ glusterd_is_brick_decommissioned (glusterd_volinfo_t *volinfo, char *hostname,
         int                     ret = -1;
 
         ret = glusterd_volume_brickinfo_get (NULL, hostname, path, volinfo,
-                                             &brickinfo, GF_PATH_COMPLETE);
+                                             &brickinfo);
         if (ret)
                 goto out;
         decommissioned = brickinfo->decommissioned;
@@ -886,20 +1107,59 @@ glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo)
 {
         GF_ASSERT (peerinfo);
         glusterd_peerctx_t      *peerctx = NULL;
+        gf_boolean_t            quorum_action = _gf_false;
+        glusterd_conf_t         *priv = THIS->private;
 
+        if (peerinfo->quorum_contrib != QUORUM_NONE)
+                quorum_action = _gf_true;
         if (peerinfo->rpc) {
+                /* cleanup the saved-frames before last unref */
+                synclock_unlock (&priv->big_lock);
+                rpc_clnt_connection_cleanup (&peerinfo->rpc->conn);
+                synclock_lock (&priv->big_lock);
+
                 peerctx = peerinfo->rpc->mydata;
                 peerinfo->rpc->mydata = NULL;
-                peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
+                peerinfo->rpc = glusterd_rpc_clnt_unref (priv, peerinfo->rpc);
                 peerinfo->rpc = NULL;
-                if (peerctx)
+                if (peerctx) {
+                        GF_FREE (peerctx->errstr);
                         GF_FREE (peerctx);
+                }
         }
         glusterd_peer_destroy (peerinfo);
 
+        if (quorum_action)
+                glusterd_do_quorum_action ();
         return 0;
 }
 
+int
+glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo)
+{
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_volinfo_t      *voliter = NULL;
+        glusterd_conf_t         *priv = NULL;
+
+        if (!volume_id)
+                return -1;
+
+        this = THIS;
+        priv = this->private;
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (uuid_compare (volume_id, voliter->volume_id))
+                        continue;
+                *volinfo = voliter;
+                ret = 0;
+                gf_log (this->name, GF_LOG_DEBUG, "Volume %s found",
+                        voliter->volname);
+                break;
+        }
+        return ret;
+}
+
 int32_t
 glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
 {
@@ -914,129 +1174,94 @@ glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
         GF_ASSERT (this);
 
         priv = this->private;
+        GF_ASSERT (priv);
 
         list_for_each_entry (tmp_volinfo, &priv->volumes, vol_list) {
                 if (!strcmp (tmp_volinfo->volname, volname)) {
-                        gf_log ("", GF_LOG_DEBUG, "Volume %s found", volname);
+                        gf_log (this->name, GF_LOG_DEBUG, "Volume %s found", volname);
                         ret = 0;
                         *volinfo = tmp_volinfo;
                         break;
                 }
         }
 
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-
 int32_t
 glusterd_service_stop (const char *service, char *pidfile, int sig,
                        gf_boolean_t force_kill)
 {
         int32_t  ret = -1;
         pid_t    pid = -1;
-        FILE    *file = NULL;
-        gf_boolean_t is_locked = _gf_false;
-
-        file = fopen (pidfile, "r+");
-
-        if (!file) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
-                                pidfile);
-                if (errno == ENOENT) {
-                        gf_log ("",GF_LOG_TRACE, "%s may not be running",
-                                service);
-                        ret = 0;
-                        goto out;
-                }
-                ret = -1;
-                goto out;
-        }
-        ret = lockf (fileno (file), F_TLOCK, 0);
-        if (!ret) {
-                is_locked = _gf_true;
-                ret = unlink (pidfile);
-                if (ret && (ENOENT != errno)) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to "
-                                "unlink stale pidfile: %s", pidfile);
-                } else if (ret && (ENOENT == errno)){
-                        ret = 0;
-                        gf_log ("", GF_LOG_INFO, "Brick already stopped");
-                }
-                goto out;
-        }
-
+        xlator_t *this = NULL;
 
-        ret = fscanf (file, "%d", &pid);
-        if (ret <= 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read pidfile: %s",
-                                pidfile);
-                ret = -1;
+        this = THIS;
+        GF_ASSERT (this);
+        if (!gf_is_service_running (pidfile, &pid)) {
+                ret = 0;
+                gf_log (this->name, GF_LOG_INFO, "%s already stopped", service);
                 goto out;
         }
-        fclose (file);
-        file = NULL;
-
-        gf_log ("", GF_LOG_INFO, "Stopping gluster %s running in pid: %d",
-                service, pid);
+        gf_log (this->name, GF_LOG_DEBUG, "Stopping gluster %s running in pid: "
+                "%d", service, pid);
 
         ret = kill (pid, sig);
-
-        if (force_kill) {
-                sleep (1);
-                file = fopen (pidfile, "r+");
-                if (!file) {
+        if (ret) {
+                switch (errno) {
+                case ESRCH:
+                        gf_log (this->name, GF_LOG_DEBUG, "%s is already stopped",
+                                service);
                         ret = 0;
                         goto out;
+                default:
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to kill %s: %s",
+                                service, strerror (errno));
                 }
-                ret = lockf (fileno (file), F_TLOCK, 0);
-                if (ret && ((EAGAIN == errno) || (EACCES == errno))) {
-                        ret = kill (pid, SIGKILL);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to "
-                                        "kill pid %d reason: %s", pid,
-                                        strerror(errno));
-                                goto out;
-                        }
+        }
+        if (!force_kill)
+                goto out;
 
-                } else if (0 == ret){
-                        is_locked = _gf_true;
-                }
-                ret = unlink (pidfile);
-                if (ret && (ENOENT != errno)) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to "
-                                "unlink pidfile: %s", pidfile);
+        sleep (1);
+        if (gf_is_service_running (pidfile, NULL)) {
+                ret = kill (pid, SIGKILL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                "kill pid %d reason: %s", pid,
+                                strerror(errno));
                         goto out;
                 }
         }
 
         ret = 0;
 out:
-        if (is_locked && file)
-                if (lockf (fileno (file), F_ULOCK, 0) < 0)
-                        gf_log ("", GF_LOG_WARNING, "Cannot unlock pidfile: %s"
-                                " reason: %s", pidfile, strerror(errno));
-        if (file)
-                fclose (file);
         return ret;
 }
 
 void
+glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len)
+{
+        char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};
+
+        md5_wrapper ((unsigned char *) sock_filepath, strlen(sock_filepath), md5_sum);
+        snprintf (sockpath, len, "%s/%s.socket", GLUSTERD_SOCK_DIR, md5_sum);
+}
+
+void
 glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
                                     glusterd_brickinfo_t *brickinfo,
                                     char *sockpath, size_t len)
 {
         char                    export_path[PATH_MAX] = {0,};
         char                    sock_filepath[PATH_MAX] = {0,};
-        char                    md5_sum[MD5_DIGEST_LEN*2+1] = {0,};
         char                    volume_dir[PATH_MAX] = {0,};
         xlator_t                *this = NULL;
         glusterd_conf_t         *priv = NULL;
         int                     expected_file_len = 0;
 
-        expected_file_len = strlen (glusterd_sock_dir) + strlen ("/") +
-                            MD5_DIGEST_LEN*2 + strlen (".socket") + 1;
+        expected_file_len = strlen (GLUSTERD_SOCK_DIR) + strlen ("/") +
+                            MD5_DIGEST_LENGTH*2 + strlen (".socket") + 1;
         GF_ASSERT (len >= expected_file_len);
         this = THIS;
         GF_ASSERT (this);
@@ -1047,10 +1272,8 @@ glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
         GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, export_path);
         snprintf (sock_filepath, PATH_MAX, "%s/run/%s-%s",
                   volume_dir, brickinfo->hostname, export_path);
-        _get_md5_str (md5_sum, sizeof (md5_sum),
-                              (uint8_t*)sock_filepath, strlen (sock_filepath));
 
-        snprintf (sockpath, len, "%s/%s.socket", glusterd_sock_dir, md5_sum);
+        glusterd_set_socket_filepath (sock_filepath, sockpath, len);
 }
 
 /* connection happens only if it is not aleady connected,
@@ -1062,8 +1285,11 @@ glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
 {
         int                     ret = 0;
         char                    socketpath[PATH_MAX] = {0};
+        char                    volume_id_str[64];
+        char                    *brickid = NULL;
         dict_t                  *options = NULL;
         struct rpc_clnt         *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
@@ -1072,44 +1298,78 @@ glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
                 glusterd_set_brick_socket_filepath (volinfo, brickinfo,
                                                     socketpath,
                                                     sizeof (socketpath));
-                ret = rpc_clnt_transport_unix_options_build (&options, socketpath);
+
+                /* Setting frame-timeout to 10mins (600seconds).
+                 * Unix domain sockets ensures that the connection is reliable.
+                 * The default timeout of 30mins used for unreliable network
+                 * connections is too long for unix domain socket connections.
+                 */
+                ret = rpc_transport_unix_options_build (&options, socketpath,
+                                                        600);
                 if (ret)
                         goto out;
+
+                uuid_utoa_r (volinfo->volume_id, volume_id_str);
+                ret = gf_asprintf (&brickid, "%s:%s:%s", volume_id_str,
+                                   brickinfo->hostname, brickinfo->path);
+                if (ret < 0)
+                        goto out;
+
+                synclock_unlock (&priv->big_lock);
                 ret = glusterd_rpc_create (&rpc, options,
                                            glusterd_brick_rpc_notify,
-                                           brickinfo);
-                if (ret)
+                                           brickid);
+                synclock_lock (&priv->big_lock);
+                if (ret) {
+                        GF_FREE (brickid);
                         goto out;
+                }
                 brickinfo->rpc = rpc;
         }
 out:
+
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+static int
+_mk_rundir_p (glusterd_volinfo_t *volinfo)
+{
+        char voldir[PATH_MAX]   = {0,};
+        char rundir[PATH_MAX]   = {0,};
+        glusterd_conf_t *priv   = NULL;
+        xlator_t        *this   = NULL;
+        int             ret     = -1;
+
+        this = THIS;
+        priv = this->private;
+        GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv);
+        snprintf (rundir, sizeof (rundir)-1, "%s/run", voldir);
+        ret =  mkdir_p (rundir, 0777, _gf_true);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create rundir");
+        return ret;
+}
+
 int32_t
 glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
-                                 glusterd_brickinfo_t  *brickinfo)
+                                 glusterd_brickinfo_t  *brickinfo,
+                                 gf_boolean_t wait)
 {
         int32_t                 ret = -1;
         xlator_t                *this = NULL;
         glusterd_conf_t         *priv = NULL;
-        char                    pidfile[PATH_MAX] = {0,};
+        char                    pidfile[PATH_MAX+1] = {0,};
         char                    volfile[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
         runner_t                runner = {0,};
-        char                    rundir[PATH_MAX] = {0,};
         char                    exp_path[PATH_MAX] = {0,};
         char                    logfile[PATH_MAX] = {0,};
         int                     port = 0;
         int                     rdma_port = 0;
-        FILE                    *file = NULL;
-        gf_boolean_t            is_locked = _gf_false;
         char                    socketpath[PATH_MAX] = {0};
         char                    glusterd_uuid[1024] = {0,};
-#ifdef DEBUG
         char                    valgrind_logfile[PATH_MAX] = {0};
-#endif
+
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
 
@@ -1117,109 +1377,73 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         GF_ASSERT (this);
 
         priv = this->private;
+        GF_ASSERT (priv);
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-        snprintf (rundir, PATH_MAX, "%s/run", path);
-        ret = mkdir (rundir, 0777);
-
-        if ((ret == -1) && (EEXIST != errno)) {
-                gf_log ("", GF_LOG_ERROR, "Unable to create rundir %s",
-                        rundir);
+        ret = _mk_rundir_p (volinfo);
+        if (ret)
                 goto out;
-        }
-
-        glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
-                                            sizeof (socketpath));
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
-                                    brickinfo->path);
-
-        file = fopen (pidfile, "r+");
-        if (file) {
-                ret = lockf (fileno (file), F_TLOCK, 0);
-                if (ret && ((EAGAIN == errno) || (EACCES == errno))) {
-                        ret = 0;
-                        gf_log ("", GF_LOG_INFO, "brick %s:%s "
-                                "already started", brickinfo->hostname,
-                                brickinfo->path);
-                        goto connect;
-                }
-        }
-
-        ret = pmap_registry_search (this, brickinfo->path,
-                                    GF_PMAP_PORT_BRICKSERVER);
-        if (ret) {
-                ret = 0;
-                file = fopen (pidfile, "r+");
-                if (file) {
-                        ret = lockf (fileno (file), F_TLOCK, 0);
-                        if (ret && ((EAGAIN == errno) || (EACCES == errno))) {
-                                ret = 0;
-                                gf_log ("", GF_LOG_INFO, "brick %s:%s "
-                                        "already started", brickinfo->hostname,
-                                        brickinfo->path);
-                                goto connect;
-                        } else if (0 == ret) {
-                                is_locked = _gf_true;
-                        }
-                }
-                /* This means, pmap has the entry, remove it */
-                ret = pmap_registry_remove (this, 0, brickinfo->path,
-                                            GF_PMAP_PORT_BRICKSERVER, NULL);
-        }
-        unlink (pidfile);
-
-        gf_log ("", GF_LOG_INFO, "About to start glusterfs"
-                " for brick %s:%s", brickinfo->hostname,
-                brickinfo->path);
-        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);
-        snprintf (volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
-                  brickinfo->hostname, exp_path);
-
-        if (!brickinfo->logfile && volinfo->logdir) {
-                snprintf (logfile, PATH_MAX, "%s/%s.log", volinfo->logdir,
-                                                          exp_path);
-                brickinfo->logfile = gf_strdup (logfile);
-        } else if (!brickinfo->logfile) {
-                snprintf (logfile, PATH_MAX, "%s/bricks/%s.log",
-                          DEFAULT_LOG_FILE_DIRECTORY, exp_path);
-                brickinfo->logfile = gf_strdup (logfile);
-        }
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+        if (gf_is_service_running (pidfile, NULL))
+                goto connect;
 
         port = brickinfo->port;
         if (!port)
                 port = pmap_registry_alloc (THIS);
 
+        /* Build the exp_path, before starting the glusterfsd even in
+           valgrind mode. Otherwise all the glusterfsd processes start
+           writing the valgrind log to the same file.
+        */
+        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);
         runinit (&runner);
 
-#ifdef DEBUG
         if (priv->valgrind) {
+                /* Run bricks with valgrind */
                 if (volinfo->logdir) {
                         snprintf (valgrind_logfile, PATH_MAX,
-                                  "%s/valgrind-%s-%s.log", volinfo->logdir,
+                                  "%s/valgrind-%s-%s.log",
+                                  volinfo->logdir,
                                   volinfo->volname, exp_path);
                 } else {
-                         snprintf (valgrind_logfile, PATH_MAX,
-                                   "%s/bricks/valgrnd-%s-%s.log",
-                                   DEFAULT_LOG_FILE_DIRECTORY,
-                                   volinfo->volname, exp_path);
+                        snprintf (valgrind_logfile, PATH_MAX,
+                                  "%s/bricks/valgrind-%s-%s.log",
+                                  DEFAULT_LOG_FILE_DIRECTORY,
+                                  volinfo->volname, exp_path);
                 }
-                /* Run bricks with valgrind */
+
                 runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                "--trace-children=yes", NULL);
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
                 runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
-	}
-#endif
+        }
+
+        snprintf (volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
+                  brickinfo->hostname, exp_path);
+
+        if (volinfo->logdir) {
+                snprintf (logfile, PATH_MAX, "%s/%s.log",
+                          volinfo->logdir, exp_path);
+        } else {
+                snprintf (logfile, PATH_MAX, "%s/bricks/%s.log",
+                          DEFAULT_LOG_FILE_DIRECTORY, exp_path);
+        }
+        if (!brickinfo->logfile)
+                brickinfo->logfile = gf_strdup (logfile);
+
+        glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
+                                            sizeof (socketpath));
+
         (void) snprintf (glusterd_uuid, 1024, "*-posix.glusterd-uuid=%s",
-                         uuid_utoa (priv->uuid));
-	runner_add_args (&runner, SBIN_DIR"/glusterfsd",
-                         "-s", "localhost", "--volfile-id", volfile,
+                         uuid_utoa (MY_UUID));
+        runner_add_args (&runner, SBIN_DIR"/glusterfsd",
+                         "-s", brickinfo->hostname, "--volfile-id", volfile,
                          "-p", pidfile, "-S", socketpath,
                          "--brick-name", brickinfo->path,
                          "-l", brickinfo->logfile,
                          "--xlator-option", glusterd_uuid,
                          NULL);
 
-	runner_add_arg (&runner, "--brick-port");
+        runner_add_arg (&runner, "--brick-port");
         if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
                 runner_argprintf (&runner, "%d", port);
         } else {
@@ -1236,26 +1460,30 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         runner_argprintf (&runner, "%s-server.listen-port=%d",
                           volinfo->volname, port);
 
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
         runner_log (&runner, "", GF_LOG_DEBUG, "Starting GlusterFS");
-        ret = runner_run (&runner);
+        if (wait) {
+                synclock_unlock (&priv->big_lock);
+                ret = runner_run (&runner);
+                synclock_lock (&priv->big_lock);
 
-        if (ret == 0) {
-                //pmap_registry_bind (THIS, port, brickinfo->path);
-                brickinfo->port = port;
-                brickinfo->rdma_port = rdma_port;
+        } else {
+                ret = runner_run_nowait (&runner);
         }
 
+        if (ret)
+                goto out;
+
+        brickinfo->port = port;
+        brickinfo->rdma_port = rdma_port;
+
 connect:
         ret = glusterd_brick_connect (volinfo, brickinfo);
         if (ret)
                 goto out;
 out:
-        if (is_locked && file)
-                if (lockf (fileno (file), F_ULOCK, 0) < 0)
-                        gf_log ("", GF_LOG_WARNING, "Cannot unlock pidfile: %s"
-                                " reason: %s", pidfile, strerror(errno));
-        if (file)
-                fclose (file);
         return ret;
 }
 
@@ -1283,7 +1511,7 @@ glusterd_brick_unlink_socket_file (glusterd_volinfo_t *volinfo,
         if (ret && (ENOENT == errno)) {
                 ret = 0;
         } else {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to remove %s"
+                gf_log (this->name, GF_LOG_ERROR, "Failed to remove %s"
                         " error: %s", socketpath, strerror (errno));
         }
 
@@ -1293,23 +1521,34 @@ glusterd_brick_unlink_socket_file (glusterd_volinfo_t *volinfo,
 int32_t
 glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo)
 {
+        rpc_clnt_t              *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
+
         GF_ASSERT (brickinfo);
 
-        if (brickinfo->rpc) {
-                rpc_clnt_unref (brickinfo->rpc);
-                brickinfo->rpc = NULL;
+        if (!brickinfo) {
+                gf_log_callingfn ("glusterd", GF_LOG_WARNING, "!brickinfo");
+                return -1;
+        }
+
+        rpc            = brickinfo->rpc;
+        brickinfo->rpc = NULL;
+
+        if (rpc) {
+                glusterd_rpc_clnt_unref (priv, rpc);
         }
+
         return 0;
 }
 
 int32_t
 glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
-                                glusterd_brickinfo_t   *brickinfo)
+                                glusterd_brickinfo_t   *brickinfo,
+                                gf_boolean_t del_brick)
 {
         xlator_t                *this = NULL;
         glusterd_conf_t         *priv = NULL;
         char                    pidfile[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
         int                     ret = 0;
 
         GF_ASSERT (volinfo);
@@ -1319,17 +1558,22 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
         GF_ASSERT (this);
 
         priv = this->private;
-        (void) glusterd_brick_disconnect (brickinfo);
+        if (del_brick)
+                list_del_init (&brickinfo->brick_list);
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
-                                    brickinfo->path);
-
-        ret = glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_false);
-        if (ret == 0) {
-                glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
-                (void) glusterd_brick_unlink_socket_file (volinfo, brickinfo);
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                (void) glusterd_brick_disconnect (brickinfo);
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+                ret = glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_false);
+                if (ret == 0) {
+                        glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
+                        (void) glusterd_brick_unlink_socket_file (volinfo, brickinfo);
+                }
         }
+
+        if (del_brick)
+                glusterd_delete_brick (volinfo, brickinfo);
+
         return ret;
 }
 
@@ -1359,149 +1603,299 @@ out:
         return ret;
 }
 
+/* Free LINE[0..N-1] and then the LINE buffer.  */
+static void
+free_lines (char **line, size_t n)
+{
+  size_t i;
+  for (i = 0; i < n; i++)
+    GF_FREE (line[i]);
+  GF_FREE (line);
+}
+
+char **
+glusterd_readin_file (const char *filepath, int *line_count)
+{
+        int         ret                    = -1;
+        int         n                      = 8;
+        int         counter                = 0;
+        char        buffer[PATH_MAX + 256] = {0};
+        char      **lines                  = NULL;
+        FILE       *fp                     = NULL;
+        void       *p;
+
+        fp = fopen (filepath, "r");
+        if (!fp)
+                goto out;
+
+        lines = GF_CALLOC (1, n * sizeof (*lines), gf_gld_mt_charptr);
+        if (!lines)
+                goto out;
+
+        for (counter = 0; fgets (buffer, sizeof (buffer), fp); counter++) {
+
+                if (counter == n-1) {
+                        n *= 2;
+                        p = GF_REALLOC (lines, n * sizeof (char *));
+                        if (!p) {
+                                free_lines (lines, n/2);
+                                lines = NULL;
+                                goto out;
+                        }
+                        lines = p;
+                }
+
+                lines[counter] = gf_strdup (buffer);
+        }
+
+        lines[counter] = NULL;
+        /* Reduce allocation to minimal size.  */
+        p = GF_REALLOC (lines, (counter + 1) * sizeof (char *));
+        if (!p) {
+                free_lines (lines, counter);
+                lines = NULL;
+                goto out;
+        }
+        lines = p;
+
+        *line_count = counter;
+        ret = 0;
+
+ out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", strerror (errno));
+        if (fp)
+                fclose (fp);
+
+        return lines;
+}
+
 int
-glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
+glusterd_compare_lines (const void *a, const void *b) {
+
+        return strcmp(* (char * const *) a, * (char * const *) b);
+}
+
+int
+glusterd_sort_and_redirect (const char *src_filepath, int dest_fd)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        char                    path[PATH_MAX] = {0,};
-        char                    cksum_path[PATH_MAX] = {0,};
-        char                    filepath[PATH_MAX] = {0,};
-        int                     fd = -1;
-        uint32_t                cksum = 0;
-        char                    buf[4096] = {0,};
+        int            ret          = -1;
+        int            line_count   = 0;
+        int            counter      = 0;
+        char         **lines        = NULL;
+
+
+        if (!src_filepath || dest_fd < 0)
+                goto out;
+
+        lines = glusterd_readin_file (src_filepath, &line_count);
+        if (!lines)
+                goto out;
+
+        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
+
+        for (counter = 0; lines[counter]; counter++) {
+
+                ret = write (dest_fd, lines[counter],
+                             strlen (lines[counter]));
+                if (ret < 0)
+                        goto out;
+
+                GF_FREE (lines[counter]);
+        }
+
+        ret = 0;
+ out:
+        GF_FREE (lines);
+
+        return ret;
+}
+
+int
+glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo, char *cksum_path,
+                               char *filepath, gf_boolean_t is_quota_conf,
+                               uint32_t *cs)
+{
+        int32_t                 ret                     = -1;
+        uint32_t                cksum                   = 0;
+        int                     fd                      = -1;
+        int                     sort_fd                 = 0;
         char                    sort_filepath[PATH_MAX] = {0};
-        gf_boolean_t            unlink_sortfile = _gf_false;
-        int                     sort_fd = 0;
-        runner_t                runner;
+        char                   *cksum_path_final        = NULL;
+        char                    buf[4096]               = {0,};
+        gf_boolean_t            unlink_sortfile         = _gf_false;
+        glusterd_conf_t        *priv                    = NULL;
+        xlator_t               *this                    = NULL;
 
         GF_ASSERT (volinfo);
-
+        this = THIS;
         priv = THIS->private;
         GF_ASSERT (priv);
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-
-        snprintf (cksum_path, sizeof (cksum_path), "%s/%s",
-                  path, GLUSTERD_CKSUM_FILE);
-
         fd = open (cksum_path, O_RDWR | O_APPEND | O_CREAT| O_TRUNC, 0600);
 
         if (-1 == fd) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to open %s, errno: %d",
-                        cksum_path, errno);
+                gf_log (this->name, GF_LOG_ERROR, "Unable to open %s,"
+                        " errno: %d", cksum_path, errno);
                 ret = -1;
                 goto out;
         }
 
-        snprintf (filepath, sizeof (filepath), "%s/%s", path,
-                  GLUSTERD_VOLUME_INFO_FILE);
-        snprintf (sort_filepath, sizeof (sort_filepath), "/tmp/%s.XXXXXX",
-                  volinfo->volname);
-        sort_fd = mkstemp (sort_filepath);
-        if (sort_fd < 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Could not generate temp file, "
-                        "reason: %s for volume: %s", strerror (errno),
-                        volinfo->volname);
-                goto out;
-        } else {
-                unlink_sortfile = _gf_true;
-        }
+        if (!is_quota_conf) {
+                snprintf (sort_filepath, sizeof (sort_filepath),
+                          "/tmp/%s.XXXXXX", volinfo->volname);
 
-        /* sort the info file, result in sort_filepath */
-        runinit (&runner);
-        runner_add_args (&runner, "sort", filepath, NULL);
-        runner_redir (&runner, STDOUT_FILENO, sort_fd);
+                sort_fd = mkstemp (sort_filepath);
+                if (sort_fd < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not generate "
+                                "temp file, reason: %s for volume: %s",
+                                strerror (errno), volinfo->volname);
+                        goto out;
+                } else {
+                        unlink_sortfile = _gf_true;
+                }
 
-        ret = runner_run (&runner);
-        close (sort_fd);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to sort file %s to %s",
-                        filepath, sort_filepath);
-                goto out;
-        }
-        ret = get_checksum_for_path (sort_filepath, &cksum);
+                /* sort the info file, result in sort_filepath */
 
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get checksum"
-                        " for path: %s", sort_filepath);
-                goto out;
+                ret = glusterd_sort_and_redirect (filepath, sort_fd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "sorting info file "
+                                "failed");
+                        goto out;
+                }
+
+                ret = close (sort_fd);
+                if (ret)
+                        goto out;
         }
 
-        snprintf (buf, sizeof (buf), "%s=%u\n", "info", cksum);
-        ret = write (fd, buf, strlen (buf));
+        cksum_path_final = is_quota_conf ? filepath : sort_filepath;
 
-        if (ret <= 0) {
-                ret = -1;
+        ret = get_checksum_for_path (cksum_path_final, &cksum);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get "
+                        "checksum for path: %s", cksum_path_final);
                 goto out;
         }
+        if (!is_quota_conf) {
+                snprintf (buf, sizeof (buf), "%s=%u\n", "info", cksum);
+                ret = write (fd, buf, strlen (buf));
+                if (ret <= 0) {
+                        ret = -1;
+                        goto out;
+                }
+        }
 
         ret = get_checksum_for_file (fd, &cksum);
-
         if (ret)
                 goto out;
 
-        volinfo->cksum = cksum;
+        *cs = cksum;
 
 out:
         if (fd > 0)
                close (fd);
         if (unlink_sortfile)
                unlink (sort_filepath);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
 
-void
-_add_volinfo_dict_to_prdict (dict_t *this, char *key, data_t *value, void *data)
+int glusterd_compute_cksum (glusterd_volinfo_t *volinfo,
+                               gf_boolean_t is_quota_conf)
 {
-        glusterd_voldict_ctx_t   *ctx = NULL;
+        int               ret                  = -1;
+        uint32_t          cs                   = 0;
+        char              cksum_path[PATH_MAX] = {0,};
+        char              path[PATH_MAX]       = {0,};
+        char              filepath[PATH_MAX]   = {0,};
+        glusterd_conf_t  *conf                 = NULL;
+        xlator_t         *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
+
+        if (is_quota_conf) {
+                snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
+                          GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+                snprintf (filepath, sizeof (filepath), "%s/%s", path,
+                          GLUSTERD_VOLUME_QUOTA_CONFIG);
+        } else {
+                snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
+                          GLUSTERD_CKSUM_FILE);
+                snprintf (filepath, sizeof (filepath), "%s/%s", path,
+                          GLUSTERD_VOLUME_INFO_FILE);
+        }
+
+        ret = glusterd_volume_compute_cksum (volinfo, cksum_path, filepath,
+                                             is_quota_conf, &cs);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to compute checksum "
+                        "for volume %s", volinfo->volname);
+                goto out;
+        }
+
+        if (is_quota_conf)
+                volinfo->quota_conf_cksum = cs;
+        else
+                volinfo->cksum = cs;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+_add_dict_to_prdict (dict_t *this, char *key, data_t *value, void *data)
+{
+        glusterd_dict_ctx_t     *ctx = NULL;
         char                    optkey[512] = {0,};
         int                     ret = -1;
 
         ctx = data;
-        snprintf (optkey, sizeof (optkey), "volume%d.%s%d", ctx->count,
+        snprintf (optkey, sizeof (optkey), "%s.%s%d", ctx->prefix,
                   ctx->key_name, ctx->opt_count);
         ret = dict_set_str (ctx->dict, optkey, key);
         if (ret)
                 gf_log ("", GF_LOG_ERROR, "option add for %s%d %s",
-                        ctx->key_name, ctx->count, key);
-        snprintf (optkey, sizeof (optkey), "volume%d.%s%d", ctx->count,
+                        ctx->key_name, ctx->opt_count, key);
+        snprintf (optkey, sizeof (optkey), "%s.%s%d", ctx->prefix,
                   ctx->val_name, ctx->opt_count);
         ret = dict_set_str (ctx->dict, optkey, value->data);
         if (ret)
                 gf_log ("", GF_LOG_ERROR, "option add for %s%d %s",
-                        ctx->val_name, ctx->count, value->data);
+                        ctx->val_name, ctx->opt_count, value->data);
         ctx->opt_count++;
 
-        return;
+        return ret;
 }
 
 int32_t
-glusterd_add_bricks_hname_path_to_dict (dict_t *dict)
+glusterd_add_bricks_hname_path_to_dict (dict_t *dict,
+                                        glusterd_volinfo_t *volinfo)
 {
-        char                    *volname = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
         glusterd_brickinfo_t    *brickinfo = NULL;
         int                     ret = 0;
         char                    key[256] = {0};
         int                     index = 0;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
 
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret)
-                goto out;
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 snprintf (key, sizeof (key), "%d-hostname", index);
                 ret = dict_set_str (dict, key, brickinfo->hostname);
+                if (ret)
+                        goto out;
+
                 snprintf (key, sizeof (key), "%d-path", index);
                 ret = dict_set_str (dict, key, brickinfo->path);
+                if (ret)
+                        goto out;
+
                 index++;
         }
 out:
@@ -1512,15 +1906,18 @@ int32_t
 glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                              dict_t  *dict, int32_t count)
 {
-        int32_t                 ret             = -1;
-        char                    key[512]        = {0,};
-        glusterd_brickinfo_t    *brickinfo      = NULL;
-        int32_t                 i               = 1;
-        char                    *volume_id_str  = NULL;
-        char                    *src_brick      = NULL;
-        char                    *dst_brick      = NULL;
-        char                    *str            = NULL;
-        glusterd_voldict_ctx_t   ctx            = {0};
+        int32_t                 ret               = -1;
+        char                    prefix[512]       = {0,};
+        char                    key[512]          = {0,};
+        glusterd_brickinfo_t    *brickinfo        = NULL;
+        int32_t                 i                 = 1;
+        char                    *volume_id_str    = NULL;
+        char                    *src_brick        = NULL;
+        char                    *dst_brick        = NULL;
+        char                    *str              = NULL;
+        glusterd_dict_ctx_t     ctx               = {0};
+        char                    *rebalance_id_str = NULL;
+        char                    *rb_id_str        = NULL;
 
         GF_ASSERT (dict);
         GF_ASSERT (volinfo);
@@ -1591,56 +1988,91 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 goto out;
 
         volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
-        if (!volume_id_str)
+        if (!volume_id_str) {
+                ret = -1;
                 goto out;
-
+        }
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.volume_id", count);
         ret = dict_set_dynstr (dict, key, volume_id_str);
         if (ret)
                 goto out;
+        volume_id_str = NULL;
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.username", count);
         str = glusterd_auth_get_username (volinfo);
-        if (!str) {
-                ret = -1;
-                goto out;
+        if (str) {
+                ret = dict_set_dynstr (dict, key, gf_strdup (str));
+                if (ret)
+                        goto out;
         }
-        ret = dict_set_dynstr (dict, key, gf_strdup (str));
-        if (ret)
-                goto out;
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.password", count);
         str = glusterd_auth_get_password (volinfo);
-        if (!str) {
+        if (str) {
+                ret = dict_set_dynstr (dict, key, gf_strdup (str));
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d.rebalance", count);
+        ret = dict_set_int32 (dict, key, volinfo->rebal.defrag_cmd);
+        if (ret)
+                goto out;
+
+        rebalance_id_str = gf_strdup (uuid_utoa
+                        (volinfo->rebal.rebalance_id));
+        if (!rebalance_id_str) {
                 ret = -1;
                 goto out;
         }
-        ret = dict_set_dynstr (dict, key, gf_strdup (str));
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d.rebalance-id", count);
+        ret = dict_set_dynstr (dict, key, rebalance_id_str);
         if (ret)
                 goto out;
+        rebalance_id_str = NULL;
 
         memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
-        ret = dict_set_int32 (dict, key, volinfo->rb_status);
+        snprintf (key, sizeof (key), "volume%d.rebalance-op", count);
+        ret = dict_set_uint32 (dict, key, volinfo->rebal.op);
         if (ret)
                 goto out;
 
+        if (volinfo->rebal.dict) {
+                snprintf (prefix, sizeof (prefix), "volume%d", count);
+                ctx.dict = dict;
+                ctx.prefix = prefix;
+                ctx.opt_count = 1;
+                ctx.key_name = "rebal-dict-key";
+                ctx.val_name = "rebal-dict-value";
+
+                dict_foreach (volinfo->rebal.dict, _add_dict_to_prdict, &ctx);
+                ctx.opt_count--;
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count);
+                ret = dict_set_int32 (dict, key, ctx.opt_count);
+                if (ret)
+                        goto out;
+        }
+
         memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.rebalance", count);
-        ret = dict_set_int32 (dict, key, volinfo->defrag_cmd);
+        snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
+        ret = dict_set_int32 (dict, key, volinfo->rep_brick.rb_status);
         if (ret)
                 goto out;
-        if (volinfo->rb_status > GF_RB_STATUS_NONE) {
+
+        if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
                           count);
                 gf_asprintf (&src_brick, "%s:%s",
-                             volinfo->src_brick->hostname,
-                             volinfo->src_brick->path);
+                             volinfo->rep_brick.src_brick->hostname,
+                             volinfo->rep_brick.src_brick->path);
                 ret = dict_set_dynstr (dict, key, src_brick);
                 if (ret)
                         goto out;
@@ -1649,21 +2081,35 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
                           count);
                 gf_asprintf (&dst_brick, "%s:%s",
-                             volinfo->dst_brick->hostname,
-                             volinfo->dst_brick->path);
+                             volinfo->rep_brick.dst_brick->hostname,
+                             volinfo->rep_brick.dst_brick->path);
                 ret = dict_set_dynstr (dict, key, dst_brick);
                 if (ret)
                         goto out;
+
+                rb_id_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
+                if (!rb_id_str) {
+                        ret = -1;
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rb_id", count);
+                ret = dict_set_dynstr (dict, key, rb_id_str);
+                if (ret)
+                        goto out;
+                rb_id_str = NULL;
         }
 
+        snprintf (prefix, sizeof (prefix), "volume%d", count);
         ctx.dict = dict;
-        ctx.count = count;
+        ctx.prefix = prefix;
         ctx.opt_count = 1;
         ctx.key_name = "key";
         ctx.val_name = "value";
         GF_ASSERT (volinfo->dict);
 
-        dict_foreach (volinfo->dict, _add_volinfo_dict_to_prdict, &ctx);
+        dict_foreach (volinfo->dict, _add_dict_to_prdict, &ctx);
         ctx.opt_count--;
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.opt-count", count);
@@ -1672,13 +2118,13 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 goto out;
 
         ctx.dict = dict;
-        ctx.count = count;
+        ctx.prefix = prefix;
         ctx.opt_count = 1;
         ctx.key_name = "slave-num";
         ctx.val_name = "slave-val";
         GF_ASSERT (volinfo->gsync_slaves);
 
-        dict_foreach (volinfo->gsync_slaves, _add_volinfo_dict_to_prdict, &ctx);
+        dict_foreach (volinfo->gsync_slaves, _add_dict_to_prdict, &ctx);
         ctx.opt_count--;
 
         memset (key, 0, sizeof (key));
@@ -1702,16 +2148,139 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 if (ret)
                         goto out;
 
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned",
+                          count, i);
+                ret = dict_set_int32 (dict, key, brickinfo->decommissioned);
+                if (ret)
+                        goto out;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.brick%d.brick_id",
+                          count, i);
+                ret = dict_set_str (dict, key, brickinfo->brick_id);
+                if (ret)
+                        goto out;
+
                 i++;
         }
 
+        /* Add volume op-versions to dict. This prevents volume inconsistencies
+         * in the cluster
+         */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.op-version", count);
+        ret = dict_set_int32 (dict, key, volinfo->op_version);
+        if (ret)
+                goto out;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.client-op-version", count);
+        ret = dict_set_int32 (dict, key, volinfo->client_op_version);
+        if (ret)
+                goto out;
+
+        /*Add volume Capability (BD Xlator) to dict*/
+        memset (key, 0 ,sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.caps", count);
+        ret = dict_set_int32 (dict, key, volinfo->caps);
 
 out:
+        GF_FREE (volume_id_str);
+        GF_FREE (rebalance_id_str);
+        GF_FREE (rb_id_str);
+
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
 }
 
+int
+glusterd_vol_add_quota_conf_to_dict (glusterd_volinfo_t *volinfo, dict_t* load,
+                                     int vol_idx)
+{
+        int   fd                    = -1;
+        char  *gfid_str             = NULL;
+        unsigned char  buf[16]      = {0};
+        char  key[PATH_MAX]         = {0};
+        int   gfid_idx              = 0;
+        int   ret                   = -1;
+        xlator_t *this              = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_store_create_quota_conf_sh_on_absence (volinfo);
+        if (ret)
+                goto out;
+
+        fd = open (volinfo->quota_conf_shandle->path, O_RDONLY);
+        if (fd == -1) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_quota_conf_skip_header (this, fd);
+        if (ret)
+                goto out;
+
+        for (gfid_idx=0; ; gfid_idx++) {
+
+                ret = read (fd, (void*)&buf, 16) ;
+                if (ret <= 0) {
+                        //Finished reading all entries in the conf file
+                        break;
+                }
+                if (ret != 16) {
+                        //This should never happen. We must have a multiple of
+                        //entry_sz bytes in our configuration file.
+                        gf_log (this->name, GF_LOG_CRITICAL, "Quota "
+                                "configuration store may be corrupt.");
+                        goto out;
+                }
+
+                gfid_str = gf_strdup (uuid_utoa (buf));
+                if (!gfid_str) {
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof(key)-1, "volume%d.gfid%d", vol_idx,
+                          gfid_idx);
+                key[sizeof(key)-1] = '\0';
+                ret = dict_set_dynstr (load, key, gfid_str);
+                if (ret) {
+                        goto out;
+                }
+
+                gfid_str = NULL;
+        }
+
+        snprintf (key, sizeof(key)-1, "volume%d.gfid-count", vol_idx);
+        key[sizeof(key)-1] = '\0';
+        ret = dict_set_int32 (load, key, gfid_idx);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof(key)-1, "volume%d.quota-cksum", vol_idx);
+        key[sizeof(key)-1] = '\0';
+        ret = dict_set_uint32 (load, key, volinfo->quota_conf_cksum);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof(key)-1, "volume%d.quota-version", vol_idx);
+        key[sizeof(key)-1] = '\0';
+        ret = dict_set_uint32 (load, key, volinfo->quota_conf_version);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        if (fd != -1)
+                close (fd);
+        GF_FREE (gfid_str);
+        return ret;
+}
+
 int32_t
 glusterd_build_volume_dict (dict_t **vols)
 {
@@ -1720,6 +2289,7 @@ glusterd_build_volume_dict (dict_t **vols)
         glusterd_conf_t         *priv = NULL;
         glusterd_volinfo_t      *volinfo = NULL;
         int32_t                 count = 0;
+        glusterd_dict_ctx_t     ctx            = {0};
 
         priv = THIS->private;
 
@@ -1733,6 +2303,11 @@ glusterd_build_volume_dict (dict_t **vols)
                 ret = glusterd_add_volume_to_dict (volinfo, dict, count);
                 if (ret)
                         goto out;
+                if (!glusterd_is_volume_quota_enabled (volinfo))
+                        continue;
+                ret = glusterd_vol_add_quota_conf_to_dict (volinfo, dict, count);
+                if (ret)
+                        goto out;
         }
 
 
@@ -1740,6 +2315,17 @@ glusterd_build_volume_dict (dict_t **vols)
         if (ret)
                 goto out;
 
+        ctx.dict = dict;
+        ctx.prefix = "global";
+        ctx.opt_count = 1;
+        ctx.key_name = "key";
+        ctx.val_name = "val";
+        dict_foreach (priv->opts, _add_dict_to_prdict, &ctx);
+        ctx.opt_count--;
+        ret = dict_set_int32 (dict, "global-opt-count", ctx.opt_count);
+        if (ret)
+                goto out;
+
         *vols = dict;
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
@@ -1750,7 +2336,8 @@ out:
 }
 
 int32_t
-glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status)
+glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status,
+                                char *hostname)
 {
 
         int32_t                 ret = -1;
@@ -1758,11 +2345,17 @@ glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status)
         glusterd_volinfo_t      *volinfo = NULL;
         char                    *volname = NULL;
         uint32_t                cksum = 0;
+        uint32_t                quota_cksum = 0;
+        uint32_t                quota_version = 0;
         int32_t                 version = 0;
+        xlator_t                *this = NULL;
 
         GF_ASSERT (vols);
         GF_ASSERT (status);
 
+        this = THIS;
+        GF_ASSERT (this);
+
         snprintf (key, sizeof (key), "volume%d.name", count);
         ret = dict_get_str (vols, key, &volname);
         if (ret)
@@ -1785,9 +2378,9 @@ glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status)
         if (version > volinfo->version) {
                 //Mismatch detected
                 ret = 0;
-                gf_log ("", GF_LOG_ERROR, "Version of volume %s differ."
-                        "local version = %d, remote version = %d",
-                        volinfo->volname, volinfo->version, version);
+                gf_log (this->name, GF_LOG_ERROR, "Version of volume %s differ."
+                        "local version = %d, remote version = %d on peer %s",
+                        volinfo->volname, volinfo->version, version, hostname);
                 *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
                 goto out;
         } else if (version < volinfo->version) {
@@ -1805,24 +2398,73 @@ glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status)
 
         if (cksum != volinfo->cksum) {
                 ret = 0;
-                gf_log ("", GF_LOG_ERROR, "Cksums of volume %s differ."
-                        " local cksum = %d, remote cksum = %d",
-                        volinfo->volname, volinfo->cksum, cksum);
+                gf_log (this->name, GF_LOG_ERROR, "Cksums of volume %s differ."
+                        " local cksum = %u, remote cksum = %u on peer %s",
+                        volinfo->volname, volinfo->cksum, cksum, hostname);
                 *status = GLUSTERD_VOL_COMP_RJT;
                 goto out;
         }
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.quota-version", count);
+        ret = dict_get_uint32 (vols, key, &quota_version);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "quota-version key absent for"
+                        " volume %s in peer %s's response", volinfo->volname,
+                        hostname);
+                ret = 0;
+        } else {
+                if (quota_version > volinfo->quota_conf_version) {
+                        //Mismatch detected
+                        ret = 0;
+                        gf_log (this->name, GF_LOG_ERROR, "Quota configuration "
+                                "versions of volume %s differ. "
+                                "local version = %d, remote version = %d "
+                                "on peer %s", volinfo->volname,
+                                volinfo->quota_conf_version, quota_version,
+                                hostname);
+                        *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
+                        goto out;
+                } else if (quota_version < volinfo->quota_conf_version) {
+                        *status = GLUSTERD_VOL_COMP_SCS;
+                        goto out;
+                }
+        }
+
+        //Now, versions are same, compare cksums.
+        //
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.quota-cksum", count);
+        ret = dict_get_uint32 (vols, key, &quota_cksum);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "quota checksum absent for "
+                        "volume %s in peer %s's response", volinfo->volname,
+                        hostname);
+                ret = 0;
+        } else {
+                if (quota_cksum != volinfo->quota_conf_cksum) {
+                        ret = 0;
+                        gf_log (this->name, GF_LOG_ERROR, "Cksums of quota "
+                                "configurations of volume %s differ. "
+                                "local cksum = %u, remote cksum = %u on "
+                                "peer %s", volinfo->volname,
+                                volinfo->quota_conf_cksum, quota_cksum,
+                                hostname);
+                        *status = GLUSTERD_VOL_COMP_RJT;
+                        goto out;
+                }
+        }
         *status = GLUSTERD_VOL_COMP_SCS;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with ret: %d, status: %d",
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with ret: %d, status: %d",
                 ret, *status);
         return ret;
 }
 
 static int32_t
-import_prdict_volinfo_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
-                            char *value_prefix, int opt_count, int count)
+import_prdict_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
+                    char *value_prefix, int opt_count, char *prefix)
 {
         char                    key[512] = {0,};
         int32_t                 ret = 0;
@@ -1834,8 +2476,8 @@ import_prdict_volinfo_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
 
         while (i <= opt_count) {
                 memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.%s%d",
-                          count, key_prefix, i);
+                snprintf (key, sizeof (key), "%s.%s%d",
+                          prefix, key_prefix, i);
                 ret = dict_get_str (vols, key, &opt_key);
                 if (ret) {
                         snprintf (msg, sizeof (msg), "Volume dict key not "
@@ -1844,8 +2486,8 @@ import_prdict_volinfo_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
                 }
 
                 memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.%s%d",
-                          count, value_prefix, i);
+                snprintf (key, sizeof (key), "%s.%s%d",
+                          prefix, value_prefix, i);
                 ret = dict_get_str (vols, key, &opt_val);
                 if (ret) {
                         snprintf (msg, sizeof (msg), "Volume dict value not "
@@ -1874,6 +2516,265 @@ out:
 
 }
 
+gf_boolean_t
+glusterd_is_quorum_option (char *option)
+{
+        gf_boolean_t    res = _gf_false;
+        int             i = 0;
+        char            *keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
+                                   GLUSTERD_QUORUM_RATIO_KEY, NULL};
+
+        for (i = 0; keys[i]; i++) {
+                if (strcmp (option, keys[i]) == 0) {
+                        res = _gf_true;
+                        break;
+                }
+        }
+        return res;
+}
+
+gf_boolean_t
+glusterd_is_quorum_changed (dict_t *options, char *option, char *value)
+{
+        int             ret = 0;
+        gf_boolean_t    reconfigured = _gf_false;
+        gf_boolean_t    all = _gf_false;
+        char            *oldquorum = NULL;
+        char            *newquorum = NULL;
+        char            *oldratio = NULL;
+        char            *newratio = NULL;
+
+        if ((strcmp ("all", option) != 0) &&
+            !glusterd_is_quorum_option (option))
+                goto out;
+
+        if (strcmp ("all", option) == 0)
+                all = _gf_true;
+
+        if (all || (strcmp (GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
+                newquorum = value;
+                ret = dict_get_str (options, GLUSTERD_QUORUM_TYPE_KEY,
+                                    &oldquorum);
+        }
+
+        if (all || (strcmp (GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
+                newratio = value;
+                ret = dict_get_str (options, GLUSTERD_QUORUM_RATIO_KEY,
+                                    &oldratio);
+        }
+
+        reconfigured = _gf_true;
+
+        if (oldquorum && newquorum && (strcmp (oldquorum, newquorum) == 0))
+                reconfigured = _gf_false;
+        if (oldratio && newratio && (strcmp (oldratio, newratio) == 0))
+                reconfigured = _gf_false;
+
+        if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
+            (newquorum == NULL))
+                reconfigured = _gf_false;
+out:
+        return reconfigured;
+}
+
+static inline gf_boolean_t
+_is_contributing_to_quorum (gd_quorum_contrib_t contrib)
+{
+        if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
+                return _gf_true;
+        return _gf_false;
+}
+
+static inline gf_boolean_t
+_does_quorum_meet (int active_count, int quorum_count)
+{
+        return (active_count >= quorum_count);
+}
+
+int
+glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
+                                    int *quorum_count)
+{
+        glusterd_peerinfo_t *peerinfo      = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        int                 ret            = -1;
+        int                 inquorum_count = 0;
+        char                *val           = NULL;
+        double              quorum_percentage = 0.0;
+        gf_boolean_t        ratio          = _gf_false;
+        int                 count          = 0;
+
+        conf = this->private;
+        //Start with counting self
+        inquorum_count = 1;
+        if (active_count)
+                *active_count = 1;
+        list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
+                if (peerinfo->quorum_contrib == QUORUM_WAITING)
+                        goto out;
+
+                if (_is_contributing_to_quorum (peerinfo->quorum_contrib))
+                        inquorum_count = inquorum_count + 1;
+
+                if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
+                        *active_count = *active_count + 1;
+        }
+
+        ret = dict_get_str (conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
+        if (ret == 0) {
+                ratio = _gf_true;
+                ret = gf_string2percent (val, &quorum_percentage);
+                if (!ret)
+                        ratio = _gf_true;
+        }
+        if (ratio)
+                count = CEILING_POS (inquorum_count *
+                                     quorum_percentage / 100.0);
+        else
+                count = (inquorum_count * 50 / 100) + 1;
+
+        *quorum_count = count;
+        ret = 0;
+out:
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo)
+{
+        gf_boolean_t    res = _gf_false;
+        char            *quorum_type = NULL;
+        int             ret = 0;
+
+        ret = dict_get_str (volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY,
+                            &quorum_type);
+        if (ret)
+                goto out;
+
+        if (strcmp (quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
+                res = _gf_true;
+out:
+        return res;
+}
+
+gf_boolean_t
+glusterd_is_any_volume_in_server_quorum (xlator_t *this)
+{
+        glusterd_conf_t         *conf = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+
+        conf = this->private;
+        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                if (glusterd_is_volume_in_server_quorum (volinfo)) {
+                        return _gf_true;
+                }
+        }
+        return _gf_false;
+}
+
+gf_boolean_t
+does_gd_meet_server_quorum (xlator_t *this)
+{
+        int                     quorum_count = 0;
+        int                     active_count   = 0;
+        gf_boolean_t            in = _gf_false;
+        glusterd_conf_t         *conf = NULL;
+        int                     ret = -1;
+
+        conf = this->private;
+        ret = glusterd_get_quorum_cluster_counts (this, &active_count,
+                                                  &quorum_count);
+        if (ret)
+                goto out;
+
+        if (!_does_quorum_meet (active_count, quorum_count)) {
+                goto out;
+        }
+
+        in = _gf_true;
+out:
+        return in;
+}
+
+int
+glusterd_spawn_daemons (void *opaque)
+{
+        glusterd_conf_t *conf = THIS->private;
+        gf_boolean_t    start_bricks = !conf->restart_done;
+
+        if (start_bricks) {
+                glusterd_restart_bricks (conf);
+                conf->restart_done = _gf_true;
+        }
+        glusterd_restart_gsyncds (conf);
+        glusterd_restart_rebalance (conf);
+        return 0;
+}
+
+void
+glusterd_do_volume_quorum_action (xlator_t *this, glusterd_volinfo_t *volinfo,
+                                  gf_boolean_t meets_quorum)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_conf_t         *conf = NULL;
+
+        conf = this->private;
+        if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                goto out;
+
+        if (!glusterd_is_volume_in_server_quorum (volinfo))
+                meets_quorum = _gf_true;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (!glusterd_is_local_brick (this, volinfo, brickinfo))
+                        continue;
+                if (meets_quorum)
+                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
+                else
+                        glusterd_brick_stop (volinfo, brickinfo, _gf_false);
+        }
+out:
+        return;
+}
+
+int
+glusterd_do_quorum_action ()
+{
+        xlator_t            *this          = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                 ret            = 0;
+        int                 active_count   = 0;
+        int                 quorum_count   = 0;
+        gf_boolean_t        meets          = _gf_false;
+
+        this = THIS;
+        conf = this->private;
+
+        conf->pending_quorum_action = _gf_true;
+        ret = glusterd_lock (conf->uuid);
+        if (ret)
+                goto out;
+
+        {
+                ret = glusterd_get_quorum_cluster_counts (this, &active_count,
+                                                          &quorum_count);
+                if (ret)
+                        goto unlock;
+
+                if (_does_quorum_meet (active_count, quorum_count))
+                        meets = _gf_true;
+                list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                        glusterd_do_volume_quorum_action (this, volinfo, meets);
+                }
+        }
+unlock:
+        (void)glusterd_unlock (conf->uuid);
+        conf->pending_quorum_action = _gf_false;
+out:
+        return ret;
+}
+
 int32_t
 glusterd_import_friend_volume_opts (dict_t *vols, int count,
                                     glusterd_volinfo_t *volinfo)
@@ -1882,6 +2783,7 @@ glusterd_import_friend_volume_opts (dict_t *vols, int count,
         int32_t                 ret = -1;
         int                     opt_count = 0;
         char                    msg[2048] = {0};
+        char                    volume_prefix[1024] = {0};
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.opt-count", count);
@@ -1892,8 +2794,9 @@ glusterd_import_friend_volume_opts (dict_t *vols, int count,
                 goto out;
         }
 
-        ret = import_prdict_volinfo_dict (vols, volinfo->dict, "key",
-                                          "value", opt_count, count);
+        snprintf (volume_prefix, sizeof (volume_prefix), "volume%d", count);
+        ret = import_prdict_dict (vols, volinfo->dict, "key", "value",
+                                  opt_count, volume_prefix);
         if (ret) {
                 snprintf (msg, sizeof (msg), "Unable to import options dict "
                           "specified for %s", volinfo->volname);
@@ -1909,9 +2812,8 @@ glusterd_import_friend_volume_opts (dict_t *vols, int count,
                 goto out;
         }
 
-        ret = import_prdict_volinfo_dict (vols, volinfo->gsync_slaves,
-                                          "slave-num", "slave-val", opt_count,
-                                          count);
+        ret = import_prdict_dict (vols, volinfo->gsync_slaves, "slave-num",
+                                  "slave-val", opt_count, volume_prefix);
         if (ret) {
                 snprintf (msg, sizeof (msg), "Unable to import gsync sessions "
                           "specified for %s", volinfo->volname);
@@ -1934,6 +2836,8 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
         int                     ret = -1;
         char                    *hostname = NULL;
         char                    *path = NULL;
+        char                    *brick_id = NULL;
+        int                     decommissioned = 0;
         glusterd_brickinfo_t    *new_brickinfo = NULL;
         char                    msg[2048] = {0};
 
@@ -1959,12 +2863,29 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
                 goto out;
         }
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.brick%d.brick_id",
+                  vol_count, brick_count);
+        ret = dict_get_str (vols, key, &brick_id);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned",
+                  vol_count, brick_count);
+        ret = dict_get_int32 (vols, key, &decommissioned);
+        if (ret) {
+                /* For backward compatibility */
+                ret = 0;
+        }
+
         ret = glusterd_brickinfo_new (&new_brickinfo);
         if (ret)
                 goto out;
 
         strcpy (new_brickinfo->path, path);
         strcpy (new_brickinfo->hostname, hostname);
+        new_brickinfo->decommissioned = decommissioned;
+        if (brick_id)
+                strcpy (new_brickinfo->brick_id, brick_id);
         //peerinfo might not be added yet
         (void) glusterd_resolve_brick (new_brickinfo);
         ret = 0;
@@ -1982,6 +2903,7 @@ glusterd_import_bricks (dict_t *vols, int32_t vol_count,
 {
         int                     ret = -1;
         int                     brick_count = 1;
+        int                     brickid = 0;
         glusterd_brickinfo_t     *new_brickinfo = NULL;
 
         GF_ASSERT (vols);
@@ -1993,6 +2915,12 @@ glusterd_import_bricks (dict_t *vols, int32_t vol_count,
                                                  &new_brickinfo);
                 if (ret)
                         goto out;
+                if (new_brickinfo->brick_id[0] == '\0')
+                        /*We were probed from a peer having op-version
+                         less than GD_OP_VER_PERSISTENT_AFR_XATTRS*/
+                        GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (new_brickinfo,
+                                                              new_volinfo,
+                                                              brickid++);
                 list_add_tail (&new_brickinfo->brick_list, &new_volinfo->bricks);
                 brick_count++;
         }
@@ -2002,6 +2930,150 @@ out:
         return ret;
 }
 
+static int
+glusterd_import_quota_conf (dict_t *vols, int vol_idx,
+                            glusterd_volinfo_t *new_volinfo)
+{
+        int     gfid_idx         = 0;
+        int     gfid_count       = 0;
+        int     ret              = -1;
+        int     fd               = -1;
+        char    key[PATH_MAX]    = {0};
+        char    *gfid_str        = NULL;
+        uuid_t   gfid            = {0,};
+        xlator_t *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!glusterd_is_volume_quota_enabled (new_volinfo)) {
+                (void) glusterd_clean_up_quota_store (new_volinfo);
+                return 0;
+        }
+
+        ret = glusterd_store_create_quota_conf_sh_on_absence (new_volinfo);
+        if (ret)
+                goto out;
+
+        fd = gf_store_mkstemp (new_volinfo->quota_conf_shandle);
+        if (fd < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key)-1, "volume%d.quota-cksum", vol_idx);
+        key[sizeof(key)-1] = '\0';
+        ret = dict_get_uint32 (vols, key, &new_volinfo->quota_conf_cksum);
+        if (ret)
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to get quota cksum");
+
+        snprintf (key, sizeof (key)-1, "volume%d.quota-version", vol_idx);
+        key[sizeof(key)-1] = '\0';
+        ret = dict_get_uint32 (vols, key, &new_volinfo->quota_conf_version);
+        if (ret)
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to get quota "
+                                                  "version");
+
+        snprintf (key, sizeof (key)-1, "volume%d.gfid-count", vol_idx);
+        key[sizeof(key)-1] = '\0';
+        ret = dict_get_int32 (vols, key, &gfid_count);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_quota_conf_stamp_header (this, fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to add header to tmp "
+                        "file");
+                goto out;
+        }
+
+        gfid_idx = 0;
+        for (gfid_idx = 0; gfid_idx < gfid_count; gfid_idx++) {
+
+                snprintf (key, sizeof (key)-1, "volume%d.gfid%d",
+                          vol_idx, gfid_idx);
+                key[sizeof(key)-1] = '\0';
+                ret = dict_get_str (vols, key, &gfid_str);
+                if (ret)
+                        goto out;
+
+                uuid_parse (gfid_str, gfid);
+                ret = write (fd, (void*)gfid, 16);
+                if (ret != 16) {
+                        gf_log (this->name, GF_LOG_CRITICAL, "Unable to write "
+                                "gfid %s into quota.conf for %s", gfid_str,
+                                new_volinfo->volname);
+                        ret = -1;
+                        goto out;
+                }
+
+        }
+
+        ret = gf_store_rename_tmppath (new_volinfo->quota_conf_shandle);
+
+        ret = 0;
+
+out:
+        if (fd != -1)
+                close (fd);
+
+        if (!ret) {
+                ret = glusterd_compute_cksum (new_volinfo, _gf_true);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_store_save_quota_version_and_cksum (new_volinfo);
+                if (ret)
+                        goto out;
+        }
+
+        if (ret && (fd > 0)) {
+                gf_store_unlink_tmppath (new_volinfo->quota_conf_shandle);
+                (void) gf_store_handle_destroy
+                                              (new_volinfo->quota_conf_shandle);
+                new_volinfo->quota_conf_shandle = NULL;
+        }
+
+        return ret;
+}
+
+int
+gd_import_friend_volume_rebal_dict (dict_t *dict, int count,
+                                    glusterd_volinfo_t *volinfo)
+{
+        int  ret        = -1;
+        char key[256]   = {0,};
+        int  dict_count  = 0;
+        char prefix[64] = {0};
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count);
+        ret = dict_get_int32 (dict, key, &dict_count);
+        if (ret) {
+                /* Older peers will not have this dict */
+                ret = 0;
+                goto out;
+        }
+
+        volinfo->rebal.dict = dict_new ();
+        if(!volinfo->rebal.dict) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (prefix, sizeof (prefix), "volume%d", count);
+        ret = import_prdict_dict (dict, volinfo->rebal.dict, "rebal-dict-key",
+                                  "rebal-dict-value", dict_count, prefix);
+out:
+        if (ret && volinfo->rebal.dict)
+                dict_unref (volinfo->rebal.dict);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_import_volinfo (dict_t *vols, int count,
                          glusterd_volinfo_t **volinfo)
@@ -2016,6 +3088,10 @@ glusterd_import_volinfo (dict_t *vols, int count,
         char               *dst_brick        = NULL;
         char               *str              = NULL;
         int                rb_status         = 0;
+        char               *rebalance_id_str = NULL;
+        char               *rb_id_str        = NULL;
+        int                op_version        = 0;
+        int                client_op_version = 0;
 
         GF_ASSERT (vols);
         GF_ASSERT (volinfo);
@@ -2104,7 +3180,8 @@ glusterd_import_volinfo (dict_t *vols, int count,
         if (ret)
                 gf_log (THIS->name, GF_LOG_INFO,
                         "peer is possibly old version");
-
+        new_volinfo->subvol_count = new_volinfo->brick_count/
+                                    glusterd_get_dist_leaf_count (new_volinfo);
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.ckusm", count);
         ret = dict_get_uint32 (vols, key, &new_volinfo->cksum);
@@ -2123,6 +3200,8 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 goto out;
         }
 
+        uuid_parse (volume_id_str, new_volinfo->volume_id);
+
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.username", count);
         ret = dict_get_str (vols, key, &str);
@@ -2152,23 +3231,49 @@ glusterd_import_volinfo (dict_t *vols, int count,
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.rebalance", count);
-        ret = dict_get_uint32 (vols, key, &new_volinfo->defrag_cmd);
+        ret = dict_get_uint32 (vols, key, &new_volinfo->rebal.defrag_cmd);
         if (ret) {
                 snprintf (msg, sizeof (msg), "%s missing in payload for %s",
                           key, volname);
                 goto out;
         }
 
-        uuid_parse (volume_id_str, new_volinfo->volume_id);
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance-id", count);
+        ret = dict_get_str (vols, key, &rebalance_id_str);
+        if (ret) {
+                /* This is not present in older glusterfs versions,
+                 * so don't error out
+                 */
+                ret = 0;
+        } else {
+                uuid_parse (rebalance_id_str, new_volinfo->rebal.rebalance_id);
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance-op", count);
+        ret = dict_get_uint32 (vols, key,(uint32_t *) &new_volinfo->rebal.op);
+        if (ret) {
+                /* This is not present in older glusterfs versions,
+                 * so don't error out
+                 */
+                ret = 0;
+        }
+        ret = gd_import_friend_volume_rebal_dict (vols, count, new_volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to import rebalance dict "
+                          "for volume.");
+                goto out;
+        }
 
         memset (key, 0, sizeof (key));
         snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
         ret = dict_get_int32 (vols, key, &rb_status);
         if (ret)
                 goto out;
-        new_volinfo->rb_status = rb_status;
+        new_volinfo->rep_brick.rb_status = rb_status;
 
-        if (new_volinfo->rb_status > GF_RB_STATUS_NONE) {
+        if (new_volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
 
                 memset (key, 0, sizeof (key));
                 snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
@@ -2177,8 +3282,8 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 if (ret)
                         goto out;
 
-                ret = glusterd_brickinfo_from_brick (src_brick,
-                                                     &new_volinfo->src_brick);
+                ret = glusterd_brickinfo_new_from_brick (src_brick,
+                                        &new_volinfo->rep_brick.src_brick);
                 if (ret) {
                         gf_log ("", GF_LOG_ERROR, "Unable to create"
                                 " src brickinfo");
@@ -2192,22 +3297,74 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 if (ret)
                         goto out;
 
-                ret = glusterd_brickinfo_from_brick (dst_brick,
-                                                     &new_volinfo->dst_brick);
+                ret = glusterd_brickinfo_new_from_brick (dst_brick,
+                                     &new_volinfo->rep_brick.dst_brick);
                 if (ret) {
                         gf_log ("", GF_LOG_ERROR, "Unable to create"
                                 " dst brickinfo");
                         goto out;
                 }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rb_id", count);
+                ret = dict_get_str (vols, key, &rb_id_str);
+                if (ret) {
+                        /* This is not present in older glusterfs versions,
+                         * so don't error out
+                         */
+                        ret = 0;
+                } else {
+                        uuid_parse (rb_id_str, new_volinfo->rep_brick.rb_id);
+                }
         }
 
 
         ret = glusterd_import_friend_volume_opts (vols, count, new_volinfo);
         if (ret)
                 goto out;
+
+        /* Import the volume's op-versions if available else set it to 1.
+         * Not having op-versions implies this informtation was obtained from a
+         * op-version 1 friend (gluster-3.3), ergo the cluster is at op-version
+         * 1 and all volumes are at op-versions 1.
+         *
+         * Either both the volume op-versions should be absent or both should be
+         * present. Only one being present is a failure
+         */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.op-version", count);
+        ret = dict_get_int32 (vols, key, &op_version);
+        if (ret)
+                ret = 0;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.client-op-version", count);
+        ret = dict_get_int32 (vols, key, &client_op_version);
+        if (ret)
+                ret = 0;
+
+        if (op_version && client_op_version) {
+                new_volinfo->op_version = op_version;
+                new_volinfo->client_op_version = client_op_version;
+        } else if (((op_version == 0) && (client_op_version != 0)) ||
+                   ((op_version != 0) && (client_op_version == 0))) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "Only one volume op-version found");
+                goto out;
+        } else {
+                new_volinfo->op_version = 1;
+                new_volinfo->client_op_version = 1;
+        }
+
+        memset (key, 0 ,sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.caps", count);
+        /*This is not present in older glusterfs versions, so ignore ret value*/
+        ret = dict_get_int32 (vols, key, &new_volinfo->caps);
+
         ret = glusterd_import_bricks (vols, count, new_volinfo);
         if (ret)
                 goto out;
+
         *volinfo = new_volinfo;
 out:
         if (msg[0])
@@ -2254,8 +3411,7 @@ glusterd_volinfo_copy_brick_portinfo (glusterd_volinfo_t *new_volinfo,
                 ret = glusterd_volume_brickinfo_get (new_brickinfo->uuid,
                                                      new_brickinfo->hostname,
                                                      new_brickinfo->path,
-                                                     old_volinfo, &old_brickinfo,
-                                                     GF_PATH_COMPLETE);
+                                                     old_volinfo, &old_brickinfo);
                 if ((0 == ret) && glusterd_is_brick_started (old_brickinfo)) {
                         new_brickinfo->port = old_brickinfo->port;
                 }
@@ -2281,10 +3437,13 @@ glusterd_volinfo_stop_stale_bricks (glusterd_volinfo_t *new_volinfo,
                 ret = glusterd_volume_brickinfo_get (old_brickinfo->uuid,
                                                      old_brickinfo->hostname,
                                                      old_brickinfo->path,
-                                                     new_volinfo, &new_brickinfo,
-                                                     GF_PATH_COMPLETE);
+                                                     new_volinfo, &new_brickinfo);
                 if (ret) {
-                        ret = glusterd_brick_stop (old_volinfo, old_brickinfo);
+                        /*TODO: may need to switch to 'atomic' flavour of
+                         * brick_stop, once we make peer rpc program also
+                         * synctask enabled*/
+                        ret = glusterd_brick_stop (old_volinfo, old_brickinfo,
+                                                   _gf_false);
                         if (ret)
                                 gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
                                         "stop brick %s:%s", old_brickinfo->hostname,
@@ -2309,7 +3468,8 @@ glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
          * stop stale bricks. Stale volume information is going to be deleted.
          * Which deletes the valid brick information inside stale volinfo.
          * We dont want brick_rpc_notify to access already deleted brickinfo.
-         * Disconnect valid bricks.
+         * Disconnect all bricks from stale_volinfo (unconditionally), since
+         * they are being deleted subsequently.
          */
         if (glusterd_is_volume_started (stale_volinfo)) {
                 if (glusterd_is_volume_started (valid_volinfo)) {
@@ -2318,10 +3478,12 @@ glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
                         //Only valid bricks will be running now.
                         (void) glusterd_volinfo_copy_brick_portinfo (valid_volinfo,
                                                                      stale_volinfo);
-                        (void) glusterd_volume_disconnect_all_bricks (stale_volinfo);
+
                 } else {
                         (void) glusterd_stop_bricks (stale_volinfo);
                 }
+
+                (void) glusterd_volume_disconnect_all_bricks (stale_volinfo);
         }
         /* Delete all the bricks and stores and vol files. They will be created
          * again by the valid_volinfo. Volume store delete should not be
@@ -2331,13 +3493,61 @@ glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
         (void) glusterd_delete_all_bricks (stale_volinfo);
         if (stale_volinfo->shandle) {
                 unlink (stale_volinfo->shandle->path);
-                (void) glusterd_store_handle_destroy (stale_volinfo->shandle);
+                (void) gf_store_handle_destroy (stale_volinfo->shandle);
                 stale_volinfo->shandle = NULL;
         }
-        (void) glusterd_volinfo_delete (stale_volinfo);
+        (void) glusterd_volinfo_remove (stale_volinfo);
         return 0;
 }
 
+/* This function updates the rebalance information of the new volinfo using the
+ * information from the old volinfo.
+ */
+int
+gd_check_and_update_rebalance_info (glusterd_volinfo_t *old_volinfo,
+                                    glusterd_volinfo_t *new_volinfo)
+{
+        int                  ret  = -1;
+        glusterd_rebalance_t *old = NULL;
+        glusterd_rebalance_t *new = NULL;
+
+        GF_ASSERT (old_volinfo);
+        GF_ASSERT (new_volinfo);
+
+        old = &(old_volinfo->rebal);
+        new = &(new_volinfo->rebal);
+
+        //Disconnect from rebalance process
+        if (old->defrag && old->defrag->rpc) {
+                rpc_transport_disconnect (old->defrag->rpc->conn.trans);
+        }
+
+        if (!uuid_is_null (old->rebalance_id) &&
+            uuid_compare (old->rebalance_id, new->rebalance_id)) {
+                (void)gd_stop_rebalance_process (old_volinfo);
+                goto out;
+        }
+
+        /* If the tasks match, copy the status and other information of the
+         * rebalance process from old_volinfo to new_volinfo
+         */
+        new->defrag_status      = old->defrag_status;
+        new->rebalance_files    = old->rebalance_files;
+        new->rebalance_data     = old->rebalance_data;
+        new->lookedup_files     = old->lookedup_files;
+        new->skipped_files      = old->skipped_files;
+        new->rebalance_failures = old->rebalance_failures;
+        new->rebalance_time     = old->rebalance_time;
+        new->dict               = (old->dict ? dict_ref (old->dict) : NULL);
+
+        /* glusterd_rebalance_t.{op, id, defrag_cmd} are copied during volume
+         * import
+         * a new defrag object should come to life with rebalance being restarted
+         */
+out:
+        return ret;
+}
+
 int32_t
 glusterd_import_friend_volume (dict_t *vols, size_t count)
 {
@@ -2360,6 +3570,8 @@ glusterd_import_friend_volume (dict_t *vols, size_t count)
 
         ret = glusterd_volinfo_find (new_volinfo->volname, &old_volinfo);
         if (0 == ret) {
+                (void) gd_check_and_update_rebalance_info (old_volinfo,
+                                                           new_volinfo);
                 (void) glusterd_delete_stale_volume (old_volinfo, new_volinfo);
         }
 
@@ -2372,7 +3584,12 @@ glusterd_import_friend_volume (dict_t *vols, size_t count)
         if (ret)
                 goto out;
 
-        list_add_tail (&new_volinfo->vol_list, &priv->volumes);
+        ret = glusterd_import_quota_conf (vols, count, new_volinfo);
+        if (ret)
+                goto out;
+
+        list_add_order (&new_volinfo->vol_list, &priv->volumes,
+                        glusterd_compare_volume_name);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning with ret: %d", ret);
         return ret;
@@ -2403,8 +3620,97 @@ out:
         return ret;
 }
 
+int
+glusterd_get_global_opt_version (dict_t *opts, uint32_t *version)
+{
+        int     ret = -1;
+        char    *version_str = NULL;
+
+        ret = dict_get_str (opts, GLUSTERD_GLOBAL_OPT_VERSION, &version_str);
+        if (ret)
+                goto out;
+
+        ret = gf_string2uint (version_str, version);
+        if (ret)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_get_next_global_opt_version_str (dict_t *opts, char **version_str)
+{
+        int             ret = -1;
+        char            version_string[64] = {0};
+        uint32_t        version = 0;
+
+        ret = glusterd_get_global_opt_version (opts, &version);
+        if (ret)
+                goto out;
+        version++;
+        snprintf (version_string, sizeof (version_string), "%"PRIu32, version);
+        *version_str = gf_strdup (version_string);
+        if (*version_str)
+                ret = 0;
+out:
+        return ret;
+}
+
 int32_t
-glusterd_compare_friend_data (dict_t  *vols, int32_t *status)
+glusterd_import_global_opts (dict_t *friend_data)
+{
+        xlator_t        *this = NULL;
+        glusterd_conf_t *conf = NULL;
+        int             ret = -1;
+        dict_t          *import_options = NULL;
+        int             count = 0;
+        uint32_t        local_version = 0;
+        uint32_t        remote_version = 0;
+
+        this = THIS;
+        conf = this->private;
+
+        ret = dict_get_int32 (friend_data, "global-opt-count", &count);
+        if (ret) {
+                //old version peer
+                ret = 0;
+                goto out;
+        }
+
+        import_options = dict_new ();
+        if (!import_options)
+                goto out;
+        ret = import_prdict_dict (friend_data, import_options, "key", "val",
+                                  count, "global");
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to import"
+                        " global options");
+                goto out;
+        }
+
+        ret = glusterd_get_global_opt_version (conf->opts, &local_version);
+        if (ret)
+                goto out;
+        ret = glusterd_get_global_opt_version (import_options, &remote_version);
+        if (ret)
+                goto out;
+        if (remote_version > local_version) {
+                ret = glusterd_store_options (this, import_options);
+                if (ret)
+                        goto out;
+                dict_unref (conf->opts);
+                conf->opts = dict_ref (import_options);
+        }
+        ret = 0;
+out:
+        if (import_options)
+                dict_unref (import_options);
+        return ret;
+}
+
+int32_t
+glusterd_compare_friend_data (dict_t  *vols, int32_t *status, char *hostname)
 {
         int32_t                 ret = -1;
         int32_t                 count = 0;
@@ -2412,6 +3718,7 @@ glusterd_compare_friend_data (dict_t  *vols, int32_t *status)
         gf_boolean_t            update = _gf_false;
         gf_boolean_t            stale_nfs = _gf_false;
         gf_boolean_t            stale_shd = _gf_false;
+        gf_boolean_t            stale_qd  = _gf_false;
 
         GF_ASSERT (vols);
         GF_ASSERT (status);
@@ -2421,7 +3728,8 @@ glusterd_compare_friend_data (dict_t  *vols, int32_t *status)
                 goto out;
 
         while (i <= count) {
-                ret = glusterd_compare_friend_volume (vols, i, status);
+                ret = glusterd_compare_friend_volume (vols, i, status,
+                                                      hostname);
                 if (ret)
                         goto out;
 
@@ -2440,6 +3748,11 @@ glusterd_compare_friend_data (dict_t  *vols, int32_t *status)
                         stale_nfs = _gf_true;
                 if (glusterd_is_nodesvc_running ("glustershd"))
                         stale_shd = _gf_true;
+                if (glusterd_is_nodesvc_running ("quotad"))
+                        stale_qd  = _gf_true;
+                ret = glusterd_import_global_opts (vols);
+                if (ret)
+                        goto out;
                 ret = glusterd_import_friend_volumes (vols);
                 if (ret)
                         goto out;
@@ -2450,6 +3763,8 @@ glusterd_compare_friend_data (dict_t  *vols, int32_t *status)
                                 glusterd_nfs_server_stop ();
                         if (stale_shd)
                                 glusterd_shd_stop ();
+                        if (stale_qd)
+                                glusterd_quotad_stop ();
                 }
         }
 
@@ -2460,40 +3775,6 @@ out:
         return ret;
 }
 
-/* Valid only in if service is 'local' to glusterd.
- * pid can be -1, if reading pidfile failed */
-gf_boolean_t
-glusterd_is_service_running (char *pidfile, int *pid)
-{
-        FILE            *file = NULL;
-        gf_boolean_t    running = _gf_false;
-        int             ret = 0;
-        int             fno = 0;
-
-        file = fopen (pidfile, "r+");
-        if (!file)
-                goto out;
-
-        fno = fileno (file);
-        ret = lockf (fno, F_TEST, 0);
-        if (ret == -1)
-                running = _gf_true;
-        if (!pid)
-                goto out;
-
-        ret = fscanf (file, "%d", pid);
-        if (ret <= 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read pidfile: %s, %s",
-                        pidfile, strerror (errno));
-                *pid = -1;
-        }
-
-out:
-        if (file)
-                fclose (file);
-        return running;
-}
-
 void
 glusterd_get_nodesvc_dir (char *server, char *workdir,
                                 char *path, size_t len)
@@ -2532,46 +3813,65 @@ glusterd_get_nodesvc_volfile (char *server, char *workdir,
         GF_ASSERT (len == PATH_MAX);
 
         glusterd_get_nodesvc_dir (server, workdir, dir, sizeof (dir));
-        snprintf (volfile, len, "%s/%s-server.vol", dir, server);
+        if (strcmp ("quotad", server) != 0)
+                snprintf (volfile, len, "%s/%s-server.vol", dir, server);
+        else
+                snprintf (volfile, len, "%s/%s.vol", dir, server);
 }
 
 void
-glusterd_shd_set_running (gf_boolean_t status)
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status)
 {
         glusterd_conf_t *priv = NULL;
 
+        GF_ASSERT (server);
         priv = THIS->private;
         GF_ASSERT (priv);
         GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
+        GF_ASSERT (priv->quotad);
 
-        priv->shd->running = status;
+        if (!strcmp("glustershd", server))
+                priv->shd->online = status;
+        else if (!strcmp ("nfs", server))
+                priv->nfs->online = status;
+        else if (!strcmp ("quotad", server))
+                priv->quotad->online = status;
 }
 
 gf_boolean_t
-glusterd_shd_is_running ()
+glusterd_is_nodesvc_online (char *server)
 {
         glusterd_conf_t *conf = NULL;
+        gf_boolean_t    online = _gf_false;
 
+        GF_ASSERT (server);
         conf = THIS->private;
         GF_ASSERT (conf);
         GF_ASSERT (conf->shd);
+        GF_ASSERT (conf->nfs);
+        GF_ASSERT (conf->quotad);
+
+        if (!strcmp (server, "glustershd"))
+                online = conf->shd->online;
+        else if (!strcmp (server, "nfs"))
+                online = conf->nfs->online;
+        else if (!strcmp (server, "quotad"))
+                online = conf->quotad->online;
 
-        return conf->shd->running;
+        return online;
 }
 
 int32_t
-glusterd_shd_set_socket_filepath (char *rundir, uuid_t uuid,
-                                  char *socketpath, int len)
+glusterd_nodesvc_set_socket_filepath (char *rundir, uuid_t uuid,
+                                      char *socketpath, int len)
 {
         char                    sockfilepath[PATH_MAX] = {0,};
-        char                    md5_str[PATH_MAX] = {0,};
 
         snprintf (sockfilepath, sizeof (sockfilepath), "%s/run-%s",
                   rundir, uuid_utoa (uuid));
-        _get_md5_str (md5_str, sizeof (md5_str),
-                      (uint8_t *)sockfilepath, sizeof (sockfilepath));
-        snprintf (socketpath, len, "%s/%s.socket", glusterd_sock_dir,
-                  md5_str);
+
+        glusterd_set_socket_filepath (sockfilepath, socketpath, len);
         return 0;
 }
 
@@ -2582,6 +3882,9 @@ glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
         glusterd_brickinfo_t    *brickinfo = NULL;
         nodesrv_t               *shd       = NULL;
         glusterd_volinfo_t      *volinfo   = NULL;
+        nodesrv_t               *nfs       = NULL;
+        nodesrv_t               *quotad    = NULL;
+
         GF_VALIDATE_OR_GOTO (THIS->name, pending_node, out);
         GF_VALIDATE_OR_GOTO (THIS->name, pending_node->node, out);
 
@@ -2595,8 +3898,16 @@ glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
 
         } else if (pending_node->type == GD_NODE_REBALANCE) {
                 volinfo = pending_node->node;
-                if (volinfo->defrag)
-                        rpc = volinfo->defrag->rpc;
+                if (volinfo->rebal.defrag)
+                        rpc = volinfo->rebal.defrag->rpc;
+
+        } else if (pending_node->type == GD_NODE_NFS) {
+                nfs = pending_node->node;
+                rpc = nfs->rpc;
+
+        } else if (pending_node->type == GD_NODE_QUOTAD) {
+                quotad = pending_node->node;
+                rpc = quotad->rpc;
 
         } else {
                 GF_ASSERT (0);
@@ -2607,19 +3918,30 @@ out:
 }
 
 struct rpc_clnt*
-glusterd_shd_get_rpc (void)
+glusterd_nodesvc_get_rpc (char *server)
 {
         glusterd_conf_t *priv   = NULL;
+        struct rpc_clnt *rpc    = NULL;
 
+        GF_ASSERT (server);
         priv = THIS->private;
         GF_ASSERT (priv);
         GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
+        GF_ASSERT (priv->quotad);
+
+        if (!strcmp (server, "glustershd"))
+                rpc = priv->shd->rpc;
+        else if (!strcmp (server, "nfs"))
+                rpc = priv->nfs->rpc;
+        else if (!strcmp (server, "quotad"))
+                rpc = priv->quotad->rpc;
 
-        return priv->shd->rpc;
+        return rpc;
 }
 
 int32_t
-glusterd_shd_set_rpc (struct rpc_clnt *rpc)
+glusterd_nodesvc_set_rpc (char *server, struct rpc_clnt *rpc)
 {
         int             ret   = 0;
         xlator_t        *this = NULL;
@@ -2630,33 +3952,68 @@ glusterd_shd_set_rpc (struct rpc_clnt *rpc)
         priv = this->private;
         GF_ASSERT (priv);
         GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
+        GF_ASSERT (priv->quotad);
 
-        priv->shd->rpc = rpc;
+        if (!strcmp ("glustershd", server))
+                priv->shd->rpc = rpc;
+        else if (!strcmp ("nfs", server))
+                priv->nfs->rpc = rpc;
+        else if (!strcmp ("quotad", server))
+                priv->quotad->rpc = rpc;
 
         return ret;
 }
 
 int32_t
-glusterd_shd_connect (char *socketpath) {
+glusterd_nodesvc_connect (char *server, char *socketpath) {
         int                     ret = 0;
         dict_t                  *options = NULL;
         struct rpc_clnt         *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
 
-        ret = rpc_clnt_transport_unix_options_build (&options, socketpath);
-        if (ret)
-                goto out;
-        ret = glusterd_rpc_create (&rpc, options,
-                                   glusterd_shd_rpc_notify,
-                                   NULL);
-        if (ret)
-                goto out;
-        (void) glusterd_shd_set_rpc (rpc);
+        rpc = glusterd_nodesvc_get_rpc (server);
+
+        if (rpc == NULL) {
+                /* Setting frame-timeout to 10mins (600seconds).
+                 * Unix domain sockets ensures that the connection is reliable.
+                 * The default timeout of 30mins used for unreliable network
+                 * connections is too long for unix domain socket connections.
+                 */
+                ret = rpc_transport_unix_options_build (&options, socketpath,
+                                                        600);
+                if (ret)
+                        goto out;
+                synclock_unlock (&priv->big_lock);
+                ret = glusterd_rpc_create (&rpc, options,
+                                           glusterd_nodesvc_rpc_notify,
+                                           server);
+                synclock_lock (&priv->big_lock);
+                if (ret)
+                        goto out;
+                (void) glusterd_nodesvc_set_rpc (server, rpc);
+        }
 out:
         return ret;
 }
 
 int32_t
-glusterd_nodesvc_start (char *server, gf_boolean_t pmap_signin)
+glusterd_nodesvc_disconnect (char *server)
+{
+        struct rpc_clnt         *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
+
+        rpc = glusterd_nodesvc_get_rpc (server);
+        (void)glusterd_nodesvc_set_rpc (server, NULL);
+
+        if (rpc)
+                glusterd_rpc_clnt_unref (priv, rpc);
+
+        return 0;
+}
+
+int32_t
+glusterd_nodesvc_start (char *server, gf_boolean_t wait)
 {
         int32_t                 ret                        = -1;
         xlator_t               *this                       = NULL;
@@ -2666,11 +4023,10 @@ glusterd_nodesvc_start (char *server, gf_boolean_t pmap_signin)
         char                    logfile[PATH_MAX]          = {0,};
         char                    volfile[PATH_MAX]          = {0,};
         char                    rundir[PATH_MAX]           = {0,};
-        char                    shd_sockfpath[PATH_MAX]    = {0,};
+        char                    sockfpath[PATH_MAX] = {0,};
         char                    volfileid[256]             = {0};
-#ifdef DEBUG
+        char                    glusterd_uuid_option[1024] = {0};
         char                    valgrind_logfile[PATH_MAX] = {0};
-#endif
 
         this = THIS;
         GF_ASSERT(this);
@@ -2702,50 +4058,59 @@ glusterd_nodesvc_start (char *server, gf_boolean_t pmap_signin)
                   server);
         snprintf (volfileid, sizeof (volfileid), "gluster/%s", server);
 
-        if (!strcmp (server, "glustershd")) {
-                glusterd_shd_set_socket_filepath (rundir,
-                                                  priv->uuid,
-                                                  shd_sockfpath,
-                                                  sizeof (shd_sockfpath));
-        }
+        glusterd_nodesvc_set_socket_filepath (rundir, MY_UUID,
+                                              sockfpath, sizeof (sockfpath));
 
         runinit (&runner);
-        //TODO: kp:change the assumption that shd is the one which signs in
-        // use runner_add_args?
-#ifdef DEBUG
+
         if (priv->valgrind) {
                 snprintf (valgrind_logfile, PATH_MAX,
-                          "%s/valgrnd-%s.log",
+                          "%s/valgrind-%s.log",
                           DEFAULT_LOG_FILE_DIRECTORY,
                           server);
 
                 runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", NULL);
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
                 runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
         }
-#endif
 
-        if (pmap_signin) {
-                runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                                 "-s", "localhost",
-                                 "--volfile-id", volfileid,
-                                 "-p", pidfile,
-                                 "-l", logfile,
-                                 "-S", shd_sockfpath, NULL);
-        } else {
-                runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                                 "-f", volfile,
-                                 "-p", pidfile,
-                                 "-l", logfile, NULL);
-        }
+        runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                         "-s", "localhost",
+                         "--volfile-id", volfileid,
+                         "-p", pidfile,
+                         "-l", logfile,
+                         "-S", sockfpath, NULL);
 
+        if (!strcmp (server, "glustershd")) {
+                snprintf (glusterd_uuid_option, sizeof (glusterd_uuid_option),
+                          "*replicate*.node-uuid=%s", uuid_utoa (MY_UUID));
+                runner_add_args (&runner, "--xlator-option",
+                                 glusterd_uuid_option, NULL);
+        }
+        if (!strcmp (server, "quotad")) {
+                runner_add_args (&runner, "--xlator-option",
+                                 "*replicate*.data-self-heal=off",
+                                 "--xlator-option",
+                                 "*replicate*.metadata-self-heal=off",
+                                 "--xlator-option",
+                                 "*replicate*.entry-self-heal=off", NULL);
+        }
         runner_log (&runner, "", GF_LOG_DEBUG,
                     "Starting the nfs/glustershd services");
 
-        ret = runner_run (&runner);
+        if (!wait) {
+                ret = runner_run_nowait (&runner);
+        } else {
+                synclock_unlock (&priv->big_lock);
+                {
+                        ret = runner_run (&runner);
+                }
+                synclock_lock (&priv->big_lock);
+        }
+
         if (ret == 0) {
-                if (pmap_signin)
-                        glusterd_shd_connect (shd_sockfpath);
+                glusterd_nodesvc_connect (server, sockfpath);
         }
 out:
         return ret;
@@ -2760,7 +4125,13 @@ glusterd_nfs_server_start ()
 int
 glusterd_shd_start ()
 {
-        return glusterd_nodesvc_start ("glustershd", _gf_true);
+        return glusterd_nodesvc_start ("glustershd", _gf_false);
+}
+
+int
+glusterd_quotad_start ()
+{
+        return glusterd_nodesvc_start ("quotad", _gf_true);
 }
 
 gf_boolean_t
@@ -2771,7 +4142,32 @@ glusterd_is_nodesvc_running (char *server)
 
         glusterd_get_nodesvc_pidfile (server, priv->workdir,
                                             pidfile, sizeof (pidfile));
-        return glusterd_is_service_running (pidfile, NULL);
+        return gf_is_service_running (pidfile, NULL);
+}
+
+int32_t
+glusterd_nodesvc_unlink_socket_file (char *server)
+{
+        int             ret = 0;
+        char            sockfpath[PATH_MAX] = {0,};
+        char            rundir[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = THIS->private;
+
+        glusterd_get_nodesvc_rundir (server, priv->workdir,
+                                     rundir, sizeof (rundir));
+
+        glusterd_nodesvc_set_socket_filepath (rundir, MY_UUID,
+                                              sockfpath, sizeof (sockfpath));
+
+        ret = unlink (sockfpath);
+        if (ret && (ENOENT == errno)) {
+                ret = 0;
+        } else {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to remove %s"
+                        " error: %s", sockfpath, strerror (errno));
+        }
+
+        return ret;
 }
 
 int32_t
@@ -2783,9 +4179,17 @@ glusterd_nodesvc_stop (char *server, int sig)
 
         if (!glusterd_is_nodesvc_running (server))
                 goto out;
+
+        (void)glusterd_nodesvc_disconnect (server);
+
         glusterd_get_nodesvc_pidfile (server, priv->workdir,
                                             pidfile, sizeof (pidfile));
         ret = glusterd_service_stop (server, pidfile, sig, _gf_true);
+
+        if (ret == 0) {
+                glusterd_nodesvc_set_online_status (server, _gf_false);
+                (void)glusterd_nodesvc_unlink_socket_file (server);
+        }
 out:
         return ret;
 }
@@ -2808,6 +4212,20 @@ glusterd_nfs_pmap_deregister ()
         else
                 gf_log ("", GF_LOG_ERROR, "De-register NFSV3 is unsuccessful");
 
+        if (pmap_unset (NLM_PROGRAM, NLMV4_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered NLM v4 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of NLM v4 failed");
+
+        if (pmap_unset (NLM_PROGRAM, NLMV1_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered NLM v1 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of NLM v1 failed");
+
+        if (pmap_unset (ACL_PROGRAM, ACLV3_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered ACL v3 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of ACL v3 failed");
 }
 
 int
@@ -2834,6 +4252,91 @@ glusterd_shd_stop ()
 }
 
 int
+glusterd_quotad_stop ()
+{
+        return glusterd_nodesvc_stop ("quotad", SIGTERM);
+}
+
+int
+glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
+                           dict_t *vol_opts)
+{
+        int                     ret = -1;
+        glusterd_conf_t         *priv = THIS->private;
+        char                    pidfile[PATH_MAX] = {0,};
+        gf_boolean_t            running = _gf_false;
+        int                     pid = -1;
+        int                     port = 0;
+        char                    key[1024] = {0,};
+
+        glusterd_get_nodesvc_pidfile (server, priv->workdir, pidfile,
+                                      sizeof (pidfile));
+        //Consider service to be running only when glusterd sees it Online
+        if (glusterd_is_nodesvc_online (server))
+                running = gf_is_service_running (pidfile, &pid);
+
+        /* For nfs-servers/self-heal-daemon setting
+         * brick<n>.hostname = "NFS Server" / "Self-heal Daemon"
+         * brick<n>.path = uuid
+         * brick<n>.port = 0
+         *
+         * This might be confusing, but cli displays the name of
+         * the brick as hostname+path, so this will make more sense
+         * when output.
+         */
+        snprintf (key, sizeof (key), "brick%d.hostname", count);
+        if (!strcmp (server, "nfs"))
+                ret = dict_set_str (dict, key, "NFS Server");
+        else if (!strcmp (server, "glustershd"))
+                ret = dict_set_str (dict, key, "Self-heal Daemon");
+        else if (!strcmp (server, "quotad"))
+                ret = dict_set_str (dict, key, "Quota Daemon");
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.path", count);
+        ret = dict_set_dynstr (dict, key, gf_strdup (uuid_utoa (MY_UUID)));
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.port", count);
+        /* Port is available only for the NFS server.
+         * Self-heal daemon doesn't provide any port for access
+         * by entities other than gluster.
+         */
+        if (!strcmp (server, "nfs")) {
+                if (dict_get (vol_opts, "nfs.port")) {
+                        ret = dict_get_int32 (vol_opts, "nfs.port", &port);
+                        if (ret)
+                                goto out;
+                } else
+                        port = GF_NFS3_PORT;
+        }
+        ret = dict_set_int32 (dict, key, port);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.pid", count);
+        ret = dict_set_int32 (dict, key, pid);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.status", count);
+        ret = dict_set_int32 (dict, key, running);
+        if (ret)
+                goto out;
+
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
 glusterd_remote_hostname_get (rpcsvc_request_t *req, char *remote_host, int len)
 {
         GF_ASSERT (req);
@@ -2861,8 +4364,7 @@ glusterd_remote_hostname_get (rpcsvc_request_t *req, char *remote_host, int len)
 
 
 out:
-        if (tmp_host)
-                GF_FREE (tmp_host);
+        GF_FREE (tmp_host);
         return ret;
 }
 
@@ -2907,6 +4409,64 @@ glusterd_reconfigure_shd ()
 }
 
 int
+glusterd_reconfigure_quotad ()
+{
+        return glusterd_reconfigure_nodesvc (glusterd_create_quotad_volfile);
+}
+
+int
+glusterd_reconfigure_nfs ()
+{
+        int             ret             = -1;
+        gf_boolean_t    identical       = _gf_false;
+
+        /*
+         * Check both OLD and NEW volfiles, if they are SAME by size
+         * and cksum i.e. "character-by-character". If YES, then
+         * NOTHING has been changed, just return.
+         */
+        ret = glusterd_check_nfs_volfile_identical (&identical);
+        if (ret)
+                goto out;
+
+        if (identical) {
+                ret = 0;
+                goto out;
+        }
+
+        /*
+         * They are not identical. Find out if the topology is changed
+         * OR just the volume options. If just the options which got
+         * changed, then inform the xlator to reconfigure the options.
+         */
+        identical = _gf_false; /* RESET the FLAG */
+        ret = glusterd_check_nfs_topology_identical (&identical);
+        if (ret)
+                goto out;
+
+        /* Topology is not changed, but just the options. But write the
+         * options to NFS volfile, so that NFS will be reconfigured.
+         */
+        if (identical) {
+                ret = glusterd_create_nfs_volfile();
+                if (ret == 0) {/* Only if above PASSES */
+                        ret = glusterd_fetchspec_notify (THIS);
+                }
+                goto out;
+        }
+
+        /*
+         * NFS volfile's topology has been changed. NFS server needs
+         * to be RESTARTED to ACT on the changed volfile.
+         */
+        ret = glusterd_check_generate_start_nfs ();
+
+out:
+        return ret;
+}
+
+
+int
 glusterd_check_generate_start_nfs ()
 {
         int ret = 0;
@@ -2931,21 +4491,52 @@ glusterd_check_generate_start_shd ()
 }
 
 int
-glusterd_nodesvcs_batch_op (glusterd_volinfo_t *volinfo,
-                             int (*nfs_op) (), int (*shd_op) ())
+glusterd_check_generate_start_quotad ()
 {
+        int ret = 0;
+
+        ret = glusterd_check_generate_start_service (glusterd_create_quotad_volfile,
+                                                     glusterd_quotad_stop,
+                                                     glusterd_quotad_start);
+        if (ret == -EINVAL)
+                ret = 0;
+        return ret;
+}
+
+int
+glusterd_nodesvcs_batch_op (glusterd_volinfo_t *volinfo, int (*nfs_op) (),
+                            int (*shd_op) (), int (*qd_op) ())
+ {
         int     ret = 0;
+        xlator_t *this = THIS;
+        glusterd_conf_t *conf = NULL;
+
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
 
         ret = nfs_op ();
         if (ret)
                 goto out;
 
-        if (volinfo && !glusterd_is_volume_replicate (volinfo))
+        if (volinfo && !glusterd_is_volume_replicate (volinfo)) {
+                ; //do nothing
+        } else {
+                ret = shd_op ();
+                if (ret)
+                        goto out;
+        }
+
+        if (conf->op_version == GD_OP_VERSION_MIN)
+                goto out;
+
+        if (volinfo && !glusterd_is_volume_quota_enabled (volinfo))
                 goto out;
 
-        ret = shd_op ();
+        ret = qd_op ();
         if (ret)
                 goto out;
+
 out:
         return ret;
 }
@@ -2955,7 +4546,8 @@ glusterd_nodesvcs_start (glusterd_volinfo_t *volinfo)
 {
         return glusterd_nodesvcs_batch_op (volinfo,
                                            glusterd_nfs_server_start,
-                                           glusterd_shd_start);
+                                           glusterd_shd_start,
+                                           glusterd_quotad_start);
 }
 
 int
@@ -2963,7 +4555,8 @@ glusterd_nodesvcs_stop (glusterd_volinfo_t *volinfo)
 {
         return glusterd_nodesvcs_batch_op (volinfo,
                                             glusterd_nfs_server_stop,
-                                            glusterd_shd_stop);
+                                            glusterd_shd_stop,
+                                            glusterd_quotad_stop);
 }
 
 gf_boolean_t
@@ -3009,29 +4602,62 @@ glusterd_all_replicate_volumes_stopped ()
         return _gf_true;
 }
 
+gf_boolean_t
+glusterd_all_volumes_with_quota_stopped ()
+{
+        glusterd_conf_t                   *priv     = NULL;
+        xlator_t                          *this     = NULL;
+        glusterd_volinfo_t                *voliter  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (!glusterd_is_volume_quota_enabled (voliter))
+                        continue;
+                if (voliter->status == GLUSTERD_STATUS_STARTED)
+                        return _gf_false;
+        }
+
+        return _gf_true;
+}
+
+
 int
 glusterd_nodesvcs_handle_graph_change (glusterd_volinfo_t *volinfo)
 {
         int (*shd_op) () = NULL;
         int (*nfs_op) () = NULL;
+        int (*qd_op)  () = NULL;
 
         shd_op = glusterd_check_generate_start_shd;
         nfs_op = glusterd_check_generate_start_nfs;
+        qd_op  = glusterd_check_generate_start_quotad;
         if (glusterd_are_all_volumes_stopped ()) {
                 shd_op = glusterd_shd_stop;
                 nfs_op = glusterd_nfs_server_stop;
-        } else if (glusterd_all_replicate_volumes_stopped()) {
-                shd_op = glusterd_shd_stop;
+                qd_op  = glusterd_quotad_stop;
+        } else {
+                if (glusterd_all_replicate_volumes_stopped()) {
+                        shd_op = glusterd_shd_stop;
+                }
+                if (glusterd_all_volumes_with_quota_stopped ()) {
+                        qd_op = glusterd_quotad_stop;
+                }
         }
-        return glusterd_nodesvcs_batch_op (volinfo, nfs_op, shd_op);
+
+        return glusterd_nodesvcs_batch_op (volinfo, nfs_op, shd_op, qd_op);
 }
 
 int
 glusterd_nodesvcs_handle_reconfigure (glusterd_volinfo_t *volinfo)
 {
         return glusterd_nodesvcs_batch_op (volinfo,
-                                           glusterd_check_generate_start_nfs,
-                                           glusterd_reconfigure_shd);
+                                           glusterd_reconfigure_nfs,
+                                           glusterd_reconfigure_shd,
+                                           glusterd_reconfigure_quotad);
 }
 
 int
@@ -3076,9 +4702,9 @@ glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path,
         list_for_each_entry (volinfo, &priv->volumes, vol_list) {
 
                 ret = glusterd_volume_brickinfo_get (uuid, hostname, path,
-                                                     volinfo, brickinfo,
-                                                     GF_PATH_COMPLETE);
-                if (!ret)
+                                                     volinfo, brickinfo);
+                if (ret == 0)
+                        /*Found*/
                         goto out;
         }
 out:
@@ -3087,7 +4713,8 @@ out:
 
 int
 glusterd_brick_start (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo)
+                      glusterd_brickinfo_t *brickinfo,
+                      gf_boolean_t wait)
 {
         int                                     ret   = -1;
         xlator_t                                *this = NULL;
@@ -3104,74 +4731,141 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo,
         if (uuid_is_null (brickinfo->uuid)) {
                 ret = glusterd_resolve_brick (brickinfo);
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "cannot resolve brick: %s:%s",
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
                                 brickinfo->hostname, brickinfo->path);
                         goto out;
                 }
         }
 
-        if (uuid_compare (brickinfo->uuid, conf->uuid)) {
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
                 ret = 0;
                 goto out;
         }
-        ret = glusterd_volume_start_glusterfs (volinfo, brickinfo);
+        ret = glusterd_volume_start_glusterfs (volinfo, brickinfo, wait);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to start "
-                        "glusterfs, ret: %d", ret);
+                gf_log (this->name, GF_LOG_ERROR, "Unable to start brick %s:%s",
+                        brickinfo->hostname, brickinfo->path);
                 goto out;
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d ", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
         return ret;
 }
 
 int
 glusterd_restart_bricks (glusterd_conf_t *conf)
 {
-        glusterd_volinfo_t       *volinfo = NULL;
-        glusterd_brickinfo_t     *brickinfo = NULL;
-        int                      ret = 0;
-        gf_boolean_t             start_nodesvcs = _gf_false;
-
-        GF_ASSERT (conf);
+        glusterd_volinfo_t   *volinfo        = NULL;
+        glusterd_brickinfo_t *brickinfo      = NULL;
+        gf_boolean_t          start_nodesvcs = _gf_false;
+        int                   ret            = 0;
 
         list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                //If volume status is not started, do not proceed
-                if (volinfo->status == GLUSTERD_STATUS_STARTED) {
-                        list_for_each_entry (brickinfo, &volinfo->bricks,
-                                             brick_list) {
-                                glusterd_brick_start (volinfo, brickinfo);
-                        }
-                        start_nodesvcs = _gf_true;
+                if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+                start_nodesvcs = _gf_true;
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
                 }
         }
+
         if (start_nodesvcs)
                 glusterd_nodesvcs_handle_graph_change (NULL);
+
         return ret;
 }
 
-void
+int
 _local_gsyncd_start (dict_t *this, char *key, data_t *value, void *data)
 {
+        char                        *path_list = NULL;
         char                        *slave = NULL;
+        char                        *slave_ip = NULL;
+        char                        *slave_vol = NULL;
+        char                        *statefile = NULL;
+        char                         buf[1024] = "faulty";
         int                          uuid_len = 0;
+        int                          ret = 0;
         char                         uuid_str[64] = {0};
-        glusterd_volinfo_t           *volinfo = NULL;
+        glusterd_volinfo_t          *volinfo = NULL;
+        char                         confpath[PATH_MAX] = "";
+        char                        *op_errstr = NULL;
+        glusterd_conf_t             *priv = NULL;
+
+        GF_ASSERT (THIS);
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (data);
 
         volinfo = data;
-        GF_ASSERT (volinfo);
         slave = strchr(value->data, ':');
         if (slave)
                 slave ++;
         else
-                return;
+                return 0;
         uuid_len = (slave - value->data - 1);
 
-
         strncpy (uuid_str, (char*)value->data, uuid_len);
-        glusterd_start_gsync (volinfo, slave, uuid_str, NULL);
+
+        /* Getting Local Brickpaths */
+        ret = glusterd_get_local_brickpaths (volinfo, &path_list);
+
+        /*Generating the conf file path needed by gsyncd */
+        ret = glusterd_get_slave_info (slave, &slave_ip,
+                                       &slave_vol, &op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (confpath, sizeof(confpath) - 1,
+                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                        priv->workdir, volinfo->volname,
+                        slave_ip, slave_vol);
+        confpath[ret] = '\0';
+
+        /* Fetching the last status of the node */
+        ret = glusterd_get_statefile_name (volinfo, slave,
+                                           confpath, &statefile);
+        if (ret) {
+                if (!strstr(slave, "::"))
+                        gf_log ("", GF_LOG_INFO,
+                                "%s is not a valid slave url.", slave);
+                else
+                        gf_log ("", GF_LOG_INFO, "Unable to get"
+                                " statefile's name");
+                goto out;
+        }
+
+        ret = glusterd_gsync_read_frm_status (statefile, buf, sizeof (buf));
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read the status");
+                goto out;
+        }
+
+        /* Looks for the last status, to find if the sessiom was running
+         * when the node went down. If the session was not started or
+         * not started, do not restart the geo-rep session */
+        if ((!strcmp (buf, "Not Started")) ||
+            (!strcmp (buf, "Stopped"))) {
+                gf_log ("", GF_LOG_INFO,
+                        "Geo-Rep Session was not started between "
+                        "%s and %s::%s. Not Restarting", volinfo->volname,
+                        slave_ip, slave_vol);
+                goto out;
+        }
+
+        glusterd_start_gsync (volinfo, slave, path_list, confpath,
+                              uuid_str, NULL);
+
+out:
+        GF_FREE (path_list);
+        GF_FREE (op_errstr);
+
+        return ret;
 }
 
 int
@@ -3195,6 +4889,15 @@ glusterd_restart_gsyncds (glusterd_conf_t *conf)
         return ret;
 }
 
+inline int
+glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
+{
+    int rcount = volinfo->replica_count;
+    int scount = volinfo->stripe_count;
+
+    return (rcount ? rcount : 1) * (scount ? scount : 1);
+}
+
 int
 glusterd_get_brickinfo (xlator_t *this, const char *brickname, int port,
                         gf_boolean_t localhost, glusterd_brickinfo_t **brickinfo)
@@ -3211,7 +4914,7 @@ glusterd_get_brickinfo (xlator_t *this, const char *brickname, int port,
         list_for_each_entry (volinfo, &priv->volumes, vol_list) {
                 list_for_each_entry (tmpbrkinfo, &volinfo->bricks,
                                      brick_list) {
-                        if (localhost && glusterd_is_local_addr (tmpbrkinfo->hostname))
+                        if (localhost && !gf_is_local_addr (tmpbrkinfo->hostname))
                                 continue;
                         if (!strcmp(tmpbrkinfo->path, brickname) &&
                             (tmpbrkinfo->port == port)) {
@@ -3223,9 +4926,23 @@ glusterd_get_brickinfo (xlator_t *this, const char *brickname, int port,
         return ret;
 }
 
+glusterd_brickinfo_t*
+glusterd_get_brickinfo_by_position (glusterd_volinfo_t *volinfo, uint32_t pos)
+{
+        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
+
+        list_for_each_entry (tmpbrkinfo, &volinfo->bricks,
+                             brick_list) {
+                if (pos == 0)
+                        return tmpbrkinfo;
+                pos--;
+        }
+        return NULL;
+}
+
 void
 glusterd_set_brick_status (glusterd_brickinfo_t  *brickinfo,
-                            gf_brick_status_t status)
+                           gf_brick_status_t status)
 {
         GF_ASSERT (brickinfo);
         brickinfo->status = status;
@@ -3273,22 +4990,23 @@ static int
 glusterd_get_brick_root (char *path, char **mount_point)
 {
         char           *ptr            = NULL;
+        char           *mnt_pt         = NULL;
         struct stat     brickstat      = {0};
         struct stat     buf            = {0};
 
         if (!path)
                 goto err;
-        *mount_point = gf_strdup (path);
-        if (!*mount_point)
+        mnt_pt = gf_strdup (path);
+        if (!mnt_pt)
                 goto err;
-        if (stat (*mount_point, &brickstat))
+        if (stat (mnt_pt, &brickstat))
                 goto err;
 
-        while ((ptr = strrchr (*mount_point, '/')) &&
-               ptr != *mount_point) {
+        while ((ptr = strrchr (mnt_pt, '/')) &&
+               ptr != mnt_pt) {
 
                 *ptr = '\0';
-                if (stat (*mount_point, &buf)) {
+                if (stat (mnt_pt, &buf)) {
                         gf_log (THIS->name, GF_LOG_ERROR, "error in "
                                 "stat: %s", strerror (errno));
                         goto err;
@@ -3300,35 +5018,58 @@ glusterd_get_brick_root (char *path, char **mount_point)
                 }
         }
 
-        if (ptr == *mount_point) {
+        if (ptr == mnt_pt) {
                 if (stat ("/", &buf)) {
                         gf_log (THIS->name, GF_LOG_ERROR, "error in "
                                 "stat: %s", strerror (errno));
                         goto err;
                 }
                 if (brickstat.st_dev == buf.st_dev)
-                        strcpy (*mount_point, "/");
+                        strcpy (mnt_pt, "/");
         }
 
+        *mount_point = mnt_pt;
         return 0;
 
  err:
-        if (*mount_point)
-                GF_FREE (*mount_point);
+        GF_FREE (mnt_pt);
         return -1;
 }
 
+static char*
+glusterd_parse_inode_size (char *stream, char *pattern)
+{
+        char *needle = NULL;
+        char *trail  = NULL;
+
+        needle = strstr (stream, pattern);
+        if (!needle)
+                goto out;
+
+        needle = nwstrtail (needle, pattern);
+
+        trail = needle;
+        while (trail && isdigit (*trail)) trail++;
+        if (trail)
+                *trail = '\0';
+
+out:
+        return needle;
+}
+
 static int
 glusterd_add_inode_size_to_dict (dict_t *dict, int count)
 {
         int             ret               = -1;
-        int             fd                = -1;
         char            key[1024]         = {0};
         char            buffer[4096]      = {0};
-        char            cmd_str[4096]     = {0};
         char           *inode_size        = NULL;
         char           *device            = NULL;
         char           *fs_name           = NULL;
+        char           *cur_word          = NULL;
+        char           *pattern           = NULL;
+        char           *trail             = NULL;
+        runner_t        runner            = {0, };
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.device", count);
@@ -3342,26 +5083,18 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count)
         if (ret)
                 goto out;
 
+        runinit (&runner);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
         /* get inode size for xfs or ext2/3/4 */
         if (!strcmp (fs_name, "xfs")) {
 
-                snprintf (cmd_str, sizeof (cmd_str),
-                          "xfs_info %s | "
-                          "grep isize | "
-                          "cut -d ' ' -f 2-  | "
-                          "cut -d '=' -f 2 | "
-                          "cut -d ' ' -f 1 "
-                          "> /tmp/gf_status.txt ",
-                          device);
+                runner_add_args (&runner, "xfs_info", device, NULL);
+                pattern = "isize=";
 
         } else if (IS_EXT_FS(fs_name)) {
 
-                snprintf (cmd_str, sizeof (cmd_str),
-                          "tune2fs -l %s | "
-                          "grep -i 'inode size' | "
-                          "awk '{print $3}' "
-                          "> /tmp/gf_status.txt ",
-                          device);
+                runner_add_args (&runner, "tune2fs", "-l", device, NULL);
+                pattern = "Inode size:";
 
         } else {
                 ret = 0;
@@ -3371,7 +5104,7 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count)
                 goto out;
         }
 
-        ret = runcmd ("/bin/sh", "-c", cmd_str, NULL);
+        ret = runner_start (&runner);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_ERROR, "could not get inode "
                         "size for %s : %s package missing", fs_name,
@@ -3380,33 +5113,42 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count)
                 goto out;
         }
 
-        fd = open ("/tmp/gf_status.txt", O_RDONLY);
-        unlink ("/tmp/gf_status.txt");
-        if (fd < 0) {
-                ret = -1;
+        for (;;) {
+                if (fgets (buffer, sizeof (buffer),
+                    runner_chio (&runner, STDOUT_FILENO)) == NULL)
+                        break;
+                trail = strrchr (buffer, '\n');
+                if (trail)
+                        *trail = '\0';
+
+                cur_word = glusterd_parse_inode_size (buffer, pattern);
+                if (cur_word)
+                        break;
+        }
+
+        ret = runner_end (&runner);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "%s exited with non-zero "
+                        "exit status", ((!strcmp (fs_name, "xfs")) ?
+                        "xfs_info" : "tune2fs"));
                 goto out;
         }
-        memset (buffer, 0, sizeof (buffer));
-        ret = read (fd, buffer, sizeof (buffer));
-        if (ret < 2) {
+        if (!cur_word) {
                 ret = -1;
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to retrieve inode "
+                        "size using %s",
+                        (!strcmp (fs_name, "xfs")? "xfs_info": "tune2fs"));
                 goto out;
         }
 
+        inode_size = gf_strdup (cur_word);
+
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "brick%d.inode_size", count);
 
-        inode_size = get_nth_word (buffer, 1);
-        if (!inode_size) {
-                ret = -1;
-                goto out;
-        }
-
         ret = dict_set_dynstr (dict, key, inode_size);
 
  out:
-        if (fd >= 0)
-                close (fd);
         if (ret)
                 gf_log (THIS->name, GF_LOG_ERROR, "failed to get inode size");
         return ret;
@@ -3432,7 +5174,12 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
         if (ret)
                 goto out;
 
-        mtab = setmntent (_PATH_MNTTAB, "r");
+        mtab = setmntent (_PATH_MOUNTED, "r");
+        if (!mtab) {
+                ret = -1;
+                goto out;
+        }
+
         entry = getmntent (mtab);
 
         while (1) {
@@ -3472,8 +5219,10 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
         ret = dict_set_dynstr (dict, key, mnt_options);
 
  out:
-        if (mnt_pt)
-                GF_FREE (mnt_pt);
+        GF_FREE (mnt_pt);
+        if (mtab)
+                endmntent (mtab);
+
         return ret;
 }
 #endif
@@ -3573,10 +5322,10 @@ glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
         int             ret                   = -1;
         int32_t         pid                   = -1;
         int32_t         brick_online          = -1;
+        char           *peer_id_str           = NULL;
         char            key[1024]             = {0};
         char            base_key[1024]        = {0};
         char            pidfile[PATH_MAX]     = {0};
-        char            path[PATH_MAX]        = {0};
         xlator_t        *this                 = NULL;
         glusterd_conf_t *priv                 = NULL;
 
@@ -3603,17 +5352,29 @@ glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        /* add peer uuid */
+        peer_id_str = gf_strdup (uuid_utoa (brickinfo->uuid));
+        if (!peer_id_str) {
+                ret = -1;
+                goto out;
+        }
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.peerid", base_key);
+        ret = dict_set_dynstr (dict, key, peer_id_str);
+        if (ret) {
+                GF_FREE (peer_id_str);
+                goto out;
+        }
+
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "%s.port", base_key);
         ret = dict_set_int32 (dict, key, brickinfo->port);
         if (ret)
                 goto out;
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, path, brickinfo->hostname,
-                                    brickinfo->path);
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
 
-        brick_online = glusterd_is_service_running (pidfile, &pid);
+        brick_online = gf_is_service_running (pidfile, &pid);
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "%s.pid", base_key);
@@ -3699,11 +5460,14 @@ glusterd_friend_find_by_uuid (uuid_t uuid,
         int                     ret = -1;
         glusterd_conf_t         *priv = NULL;
         glusterd_peerinfo_t     *entry = NULL;
+        xlator_t                *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (peerinfo);
 
         *peerinfo = NULL;
-        priv    = THIS->private;
+        priv    = this->private;
 
         GF_ASSERT (priv);
 
@@ -3713,7 +5477,7 @@ glusterd_friend_find_by_uuid (uuid_t uuid,
         list_for_each_entry (entry, &priv->peers, uuid_list) {
                 if (!uuid_compare (entry->uuid, uuid)) {
 
-                        gf_log ("glusterd", GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                  "Friend found... state: %s",
                         glusterd_friend_sm_state_name_get (entry->state.state));
                         *peerinfo = entry;
@@ -3721,7 +5485,7 @@ glusterd_friend_find_by_uuid (uuid_t uuid,
                 }
         }
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Friend with uuid: %s, not found",
+        gf_log (this->name, GF_LOG_DEBUG, "Friend with uuid: %s, not found",
                 uuid_utoa (uuid));
         return ret;
 }
@@ -3832,23 +5596,25 @@ glusterd_hostname_to_uuid (char *hostname, uuid_t uuid)
 
         ret = glusterd_friend_find_by_hostname (hostname, &peerinfo);
         if (ret) {
-                ret = glusterd_is_local_addr (hostname);
-                if (ret)
+                if (gf_is_local_addr (hostname)) {
+                        uuid_copy (uuid, MY_UUID);
+                        ret = 0;
+                } else {
                         goto out;
-                else
-                        uuid_copy (uuid, priv->uuid);
+                }
         } else {
                 uuid_copy (uuid, peerinfo->uuid);
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
 int
 glusterd_brick_stop (glusterd_volinfo_t *volinfo,
-                     glusterd_brickinfo_t *brickinfo)
+                     glusterd_brickinfo_t *brickinfo,
+                     gf_boolean_t del_brick)
 {
         int                                     ret   = -1;
         xlator_t                                *this = NULL;
@@ -3865,38 +5631,39 @@ glusterd_brick_stop (glusterd_volinfo_t *volinfo,
         if (uuid_is_null (brickinfo->uuid)) {
                 ret = glusterd_resolve_brick (brickinfo);
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "cannot resolve brick: %s:%s",
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
                                 brickinfo->hostname, brickinfo->path);
                         goto out;
                 }
         }
 
-        if (uuid_compare (brickinfo->uuid, conf->uuid)) {
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
                 ret = 0;
+                if (del_brick)
+                        glusterd_delete_brick (volinfo, brickinfo);
                 goto out;
         }
 
-        gf_log ("", GF_LOG_INFO, "About to stop glusterfs"
+        gf_log (this->name, GF_LOG_DEBUG, "About to stop glusterfs"
                 " for brick %s:%s", brickinfo->hostname,
                 brickinfo->path);
-        ret = glusterd_volume_stop_glusterfs (volinfo, brickinfo);
+        ret = glusterd_volume_stop_glusterfs (volinfo, brickinfo, del_brick);
         if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Unable to remove"
+                gf_log (this->name, GF_LOG_CRITICAL, "Unable to stop"
                         " brick: %s:%s", brickinfo->hostname,
                         brickinfo->path);
                 goto out;
         }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d ", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
         return ret;
 }
 
 int
 glusterd_is_defrag_on (glusterd_volinfo_t *volinfo)
 {
-        return (volinfo->defrag != NULL);
+        return (volinfo->rebal.defrag != NULL);
 }
 
 gf_boolean_t
@@ -3918,7 +5685,6 @@ glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
                              char *op_errstr, size_t len)
 {
         glusterd_brickinfo_t    *newbrickinfo = NULL;
-        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
         int                     ret = -1;
         gf_boolean_t            is_allocated = _gf_false;
         glusterd_peerinfo_t     *peerinfo = NULL;
@@ -3935,7 +5701,7 @@ glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
         GF_ASSERT (op_errstr);
 
         if (!brickinfo) {
-                ret = glusterd_brickinfo_from_brick (brick, &newbrickinfo);
+                ret = glusterd_brickinfo_new_from_brick (brick, &newbrickinfo);
                 if (ret)
                         goto out;
                 is_allocated = _gf_true;
@@ -3945,42 +5711,54 @@ glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
 
         ret = glusterd_resolve_brick (newbrickinfo);
         if (ret) {
-                snprintf (op_errstr, len, "Host %s not a friend",
-                          newbrickinfo->hostname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", op_errstr);
+                snprintf(op_errstr, len, "Host %s is not in \'Peer "
+                         "in Cluster\' state", newbrickinfo->hostname);
                 goto out;
         }
 
-        if (!uuid_compare (priv->uuid, newbrickinfo->uuid))
-                goto brick_validation;
-        ret = glusterd_friend_find_by_uuid (newbrickinfo->uuid, &peerinfo);
-        if (ret)
-                goto out;
-        if ((!peerinfo->connected) ||
-            (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)) {
-                snprintf(op_errstr, len, "Host %s not connected",
-                         newbrickinfo->hostname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", op_errstr);
-                ret = -1;
-                goto out;
-        }
-brick_validation:
-        ret = glusterd_brickinfo_get (newbrickinfo->uuid,
-                                      newbrickinfo->hostname,
-                                      newbrickinfo->path, &tmpbrkinfo);
-        if (!ret) {
-                snprintf(op_errstr, len, "Brick: %s already in use",
-                         brick);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", op_errstr);
-                ret = -1;
-                goto out;
+        if (!uuid_compare (MY_UUID, newbrickinfo->uuid)) {
+                /* brick is local */
+                if (!glusterd_is_brickpath_available (newbrickinfo->uuid,
+                                                      newbrickinfo->path)) {
+                        snprintf(op_errstr, len, "Brick: %s not available."
+                                 " Brick may be containing or be contained "
+                                 "by an existing brick", brick);
+                        ret = -1;
+                        goto out;
+                }
+
         } else {
-                ret = 0;
+                ret = glusterd_friend_find_by_uuid (newbrickinfo->uuid,
+                                                    &peerinfo);
+                if (ret) {
+                        snprintf (op_errstr, len, "Failed to find host %s",
+                                  newbrickinfo->hostname);
+                        goto out;
+                }
+
+                if ((!peerinfo->connected)) {
+                        snprintf(op_errstr, len, "Host %s not connected",
+                                 newbrickinfo->hostname);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) {
+                        snprintf(op_errstr, len, "Host %s is not in \'Peer "
+                                 "in Cluster\' state",
+                                 newbrickinfo->hostname);
+                        ret = -1;
+                        goto out;
+                }
         }
+
+        ret = 0;
 out:
-        if (is_allocated && newbrickinfo)
+        if (is_allocated)
                 glusterd_brickinfo_delete (newbrickinfo);
-        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d ", ret);
+        if (op_errstr[0] != '\0')
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
         return ret;
 }
 
@@ -3988,8 +5766,8 @@ int
 glusterd_is_rb_started(glusterd_volinfo_t *volinfo)
 {
         gf_log ("", GF_LOG_DEBUG,
-                "is_rb_started:status=%d", volinfo->rb_status);
-        return (volinfo->rb_status == GF_RB_STATUS_STARTED);
+                "is_rb_started:status=%d", volinfo->rep_brick.rb_status);
+        return (volinfo->rep_brick.rb_status == GF_RB_STATUS_STARTED);
 
 }
 
@@ -3997,9 +5775,9 @@ int
 glusterd_is_rb_paused ( glusterd_volinfo_t *volinfo)
 {
         gf_log ("", GF_LOG_DEBUG,
-                "is_rb_paused:status=%d", volinfo->rb_status);
+                "is_rb_paused:status=%d", volinfo->rep_brick.rb_status);
 
-        return (volinfo->rb_status == GF_RB_STATUS_PAUSED);
+        return (volinfo->rep_brick.rb_status == GF_RB_STATUS_PAUSED);
 }
 
 inline int
@@ -4007,10 +5785,10 @@ glusterd_set_rb_status (glusterd_volinfo_t *volinfo, gf_rb_status_t status)
 {
         gf_log ("", GF_LOG_DEBUG,
                 "setting status from %d to %d",
-                volinfo->rb_status,
+                volinfo->rep_brick.rb_status,
                 status);
 
-        volinfo->rb_status = status;
+        volinfo->rep_brick.rb_status = status;
         return 0;
 }
 
@@ -4018,142 +5796,192 @@ inline int
 glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
                           glusterd_brickinfo_t *src, glusterd_brickinfo_t *dst)
 {
-        if (!volinfo->src_brick || !volinfo->dst_brick)
+        glusterd_replace_brick_t        *rb = NULL;
+
+        GF_ASSERT (volinfo);
+
+        rb = &volinfo->rep_brick;
+
+        if (!rb->src_brick || !rb->dst_brick)
                 return -1;
 
-        if (strcmp (volinfo->src_brick->hostname, src->hostname) ||
-            strcmp (volinfo->src_brick->path, src->path)) {
+        if (strcmp (rb->src_brick->hostname, src->hostname) ||
+            strcmp (rb->src_brick->path, src->path)) {
                 gf_log("", GF_LOG_ERROR, "Replace brick src bricks differ");
                 return -1;
         }
-        if (strcmp (volinfo->dst_brick->hostname, dst->hostname) ||
-            strcmp (volinfo->dst_brick->path, dst->path)) {
+
+        if (strcmp (rb->dst_brick->hostname, dst->hostname) ||
+            strcmp (rb->dst_brick->path, dst->path)) {
                 gf_log ("", GF_LOG_ERROR, "Replace brick dst bricks differ");
                 return -1;
         }
+
         return 0;
 }
 
-int
-glusterd_brick_create_path (char *host, char *path, uuid_t uuid, mode_t mode,
-                            char **op_errstr)
+/*path needs to be absolute; works only on gfid, volume-id*/
+static int
+glusterd_is_uuid_present (char *path, char *xattr, gf_boolean_t *present)
 {
-        int     ret = -1;
-        char    msg[2048] = {0};
-        struct  stat st_buf = {0};
-        uuid_t  gfid = {0,};
-        uuid_t  old_uuid = {0,};
-        char    old_uuid_buf[64] = {0,};
-
-        ret = stat (path, &st_buf);
-        if ((!ret) && (!S_ISDIR (st_buf.st_mode))) {
-                snprintf (msg, sizeof (msg), "brick %s:%s, "
-                          "path %s is not a directory", host, path, path);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                ret = -1;
+        GF_ASSERT (path);
+        GF_ASSERT (xattr);
+        GF_ASSERT (present);
+
+        int     ret      = -1;
+        uuid_t  uid     = {0,};
+
+        if (!path || !xattr || !present)
                 goto out;
-        } else if (!ret) {
-                goto check_xattr;
-        } else {
-                ret = mkdir (path, mode);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "brick: %s:%s, path "
-                                  "creation failed, reason: %s",
-                                  host, path, strerror(errno));
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+
+        ret = sys_lgetxattr (path, xattr, &uid, 16);
+
+        if (ret >= 0) {
+                *present = _gf_true;
+                ret = 0;
+                goto out;
+        }
+
+        switch (errno) {
+#if defined(ENODATA)
+                case ENODATA: /* FALLTHROUGH */
+#endif
+#if defined(ENOATTR) && (ENOATTR != ENODATA)
+                case ENOATTR: /* FALLTHROUGH */
+#endif
+                case ENOTSUP:
+                        *present = _gf_false;
+                        ret = 0;
+                        break;
+                default:
+                        break;
+        }
+out:
+        return ret;
+}
+
+/*path needs to be absolute*/
+static int
+glusterd_is_path_in_use (char *path, gf_boolean_t *in_use, char **op_errstr)
+{
+        int             i               = 0;
+        int             ret             = -1;
+        gf_boolean_t    used            = _gf_false;
+        char            dir[PATH_MAX]   = {0,};
+        char            *curdir         = NULL;
+        char            msg[2048]       = {0};
+        char            *keys[3]         = {GFID_XATTR_KEY,
+                                            GF_XATTR_VOL_ID_KEY,
+                                            NULL};
+
+        GF_ASSERT (path);
+        if (!path)
+                goto out;
+
+        strcpy (dir, path);
+        curdir = dir;
+        do {
+                for (i = 0; !used && keys[i]; i++) {
+                        ret = glusterd_is_uuid_present (curdir, keys[i], &used);
+                        if (ret)
+                                goto out;
+                }
+
+                if (used)
+                        break;
+
+                curdir = dirname (curdir);
+                if (!strcmp (curdir, "."))
                         goto out;
+
+
+        } while (strcmp (curdir, "/"));
+
+        if (!strcmp (curdir, "/")) {
+                for (i = 0; !used && keys[i]; i++) {
+                        ret = glusterd_is_uuid_present (curdir, keys[i], &used);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+        ret = 0;
+        *in_use = used;
+out:
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get extended "
+                          "attribute %s, reason: %s", keys[i],
+                          strerror (errno));
+        }
+
+        if (*in_use) {
+                if (!strcmp (path, curdir)) {
+                        snprintf (msg, sizeof (msg), "%s is already part of a "
+                          "volume", path);
                 } else {
-                        goto check_xattr;
+                        snprintf (msg, sizeof (msg), "parent directory %s is "
+                          "already part of a volume", curdir);
                 }
         }
 
-/* To check if filesystem is read-only
-   and if it supports extended attributes */
-check_xattr:
+        if (strlen (msg)) {
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+
+        return ret;
+}
+
+int
+glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
+                                    char **op_errstr, gf_boolean_t is_force)
+{
+        int             ret             = -1;
+        char            msg[2048]       = {0,};
+        gf_boolean_t    in_use          = _gf_false;
+        int             flags           = 0;
+
+        /* Check for xattr support in backend fs */
         ret = sys_lsetxattr (path, "trusted.glusterfs.test",
                              "working", 8, 0);
         if (ret) {
-                snprintf (msg, sizeof (msg), "glusterfs is not"
+                snprintf (msg, sizeof (msg), "Glusterfs is not"
                           " supported on brick: %s:%s.\nSetting"
                           " extended attributes failed, reason:"
                           " %s.", host, path, strerror(errno));
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
                 goto out;
+
         } else {
-                /* Remove xattr *cannot* fail after setting it succeeded */
                 sys_lremovexattr (path, "trusted.glusterfs.test");
         }
 
-        /* Now check if the export directory has some other 'gfid',
-           other than that of root '/' */
-        ret = sys_lgetxattr (path, "trusted.gfid", gfid, 16);
-        if (ret == 16) {
-                if (!__is_root_gfid (gfid)) {
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "%s: gfid (%s) is not that of glusterfs '/' ",
-                                path, uuid_utoa (gfid));
-                        snprintf (msg, sizeof (msg),
-                                  "'%s:%s' gfid (%s) is not that of "
-                                  "glusterfs '/' ", host, path, uuid_utoa (gfid));
-                        ret = -1;
-                        goto out;
-                }
-        } else if (ret != -1) {
-                /* Wrong 'gfid' is set, it should be error */
-                ret = -1;
-                snprintf (msg, sizeof (msg), "'%s:%s' has wrong entry"
-                          "for 'gfid'.", host, path);
-                goto out;
-        } else if ((ret == -1) && (errno != ENODATA)) {
-                /* Wrong 'gfid' is set, it should be error */
-                snprintf (msg, sizeof (msg), "'%s:%s' has failed to fetch "
-                          "'gfid' (%s)", host, path, strerror (errno));
+        ret = glusterd_is_path_in_use (path, &in_use, op_errstr);
+        if (ret)
                 goto out;
-        }
 
-        ret = 0;
-        if (!uuid)
-                goto out;
-
-        /* This 'key' is set when the volume is started for the first time */
-        ret = sys_lgetxattr (path, "trusted.glusterfs.volume-id",
-                             old_uuid, 16);
-        if (ret == 16) {
-                if (uuid_compare (old_uuid, uuid)) {
-                        uuid_utoa_r (old_uuid, old_uuid_buf);
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "%s: mismatching volume-id (%s) recieved. "
-                                "already is a part of volume %s ",
-                                path, uuid_utoa (uuid), old_uuid_buf);
-                        snprintf (msg, sizeof (msg), "'%s:%s' has been part of "
-                                  "a deleted volume with id %s. Please "
-                                  "re-create the brick directory.",
-                                  host, path, old_uuid_buf);
-                        ret = -1;
-                        goto out;
-                }
-        } else if (ret != -1) {
-                /* Wrong 'volume-id' is set, it should be error */
+        if (in_use && !is_force) {
                 ret = -1;
-                snprintf (msg, sizeof (msg), "'%s:%s' has wrong entry"
-                          "for 'volume-id'.", host, path);
-                goto out;
-        } else if ((ret == -1) && (errno != ENODATA)) {
-                /* Wrong 'volume-id' is set, it should be error */
-                snprintf (msg, sizeof (msg), "'%s:%s' : failed to fetch "
-                          "'volume-id' (%s)", host, path, strerror (errno));
                 goto out;
+        }
 
+
+        if (!is_force)
+                flags = XATTR_CREATE;
+
+        ret = sys_lsetxattr (path, GF_XATTR_VOL_ID_KEY, uuid, 16,
+                             flags);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to set extended "
+                          "attributes %s, reason: %s",
+                          GF_XATTR_VOL_ID_KEY, strerror (errno));
+                goto out;
         }
-        /* if 'ret == -1' then 'volume-id' not set, seems to be a fresh
-           directory */
 
         ret = 0;
 out:
-        if (msg[0] != '\0')
+        if (strlen (msg))
                 *op_errstr = gf_strdup (msg);
 
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
@@ -4164,9 +5992,8 @@ glusterd_sm_tr_log_transition_add_to_dict (dict_t *dict,
 {
         int     ret = -1;
         char    key[512] = {0};
-	char    timestr[256] = {0,};
+	char    timestr[64] = {0,};
         char    *str = NULL;
-	struct tm   tm = {0};
 
         GF_ASSERT (dict);
         GF_ASSERT (log);
@@ -4195,9 +6022,8 @@ glusterd_sm_tr_log_transition_add_to_dict (dict_t *dict,
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "log%d-time", count);
-	localtime_r ((const time_t*)&log->transitions[i].time, &tm);
-        memset (timestr, 0, sizeof (timestr));
-	strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", &tm);
+        gf_time_fmt (timestr, sizeof timestr, log->transitions[i].time,
+                     gf_timefmt_FT);
         str = gf_strdup (timestr);
         ret = dict_set_dynstr (dict, key, str);
         if (ret)
@@ -4285,8 +6111,7 @@ glusterd_sm_tr_log_delete (glusterd_sm_tr_log_t *log)
 {
         if (!log)
                 return;
-        if (log->transitions)
-                GF_FREE (log->transitions);
+        GF_FREE (log->transitions);
         return;
 }
 
@@ -4298,6 +6123,10 @@ glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
         glusterd_sm_transition_t *transitions = NULL;
         int                      ret = -1;
         int                      next = 0;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (log);
         if (!log)
@@ -4320,18 +6149,18 @@ glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
         if (log->count < log->size)
                 log->count++;
         ret = 0;
-        gf_log ("glusterd", GF_LOG_DEBUG, "Transitioning from '%s' to '%s' "
+        gf_log (this->name, GF_LOG_DEBUG, "Transitioning from '%s' to '%s' "
                 "due to event '%s'", log->state_name_get (old_state),
                 log->state_name_get (new_state), log->event_name_get (event));
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
 int
 glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
-                       glusterd_friend_sm_state_t state,
-                       uuid_t *uuid, const char *hostname)
+                       glusterd_friend_sm_state_t state, uuid_t *uuid,
+                       const char *hostname, int port)
 {
         glusterd_peerinfo_t      *new_peer = NULL;
         int                      ret = -1;
@@ -4361,6 +6190,9 @@ glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
         if (ret)
                 goto out;
 
+        if (new_peer->state.state == GD_FRIEND_STATE_BEFRIENDED)
+                new_peer->quorum_contrib = QUORUM_WAITING;
+        new_peer->port = port;
         *peerinfo = new_peer;
 out:
         if (ret && new_peer)
@@ -4384,8 +6216,7 @@ glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo)
         }
 
         list_del_init (&peerinfo->uuid_list);
-        if (peerinfo->hostname)
-                GF_FREE (peerinfo->hostname);
+        GF_FREE (peerinfo->hostname);
         glusterd_sm_tr_log_delete (&peerinfo->sm_log);
         GF_FREE (peerinfo);
         peerinfo = NULL;
@@ -4412,7 +6243,7 @@ glusterd_remove_pending_entry (struct list_head *list, void *elem)
                 }
         }
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 
 }
@@ -4452,9 +6283,9 @@ glusterd_delete_volume (glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        ret = glusterd_volinfo_delete (volinfo);
+        glusterd_volinfo_remove (volinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
@@ -4463,18 +6294,15 @@ glusterd_delete_brick (glusterd_volinfo_t* volinfo,
                        glusterd_brickinfo_t *brickinfo)
 {
         int             ret = 0;
+        char      voldir[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = THIS->private;
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
 
-#ifdef DEBUG
-        ret = glusterd_volume_brickinfo_get (brickinfo->uuid,
-                                             brickinfo->hostname,
-                                             brickinfo->path, volinfo,
-                                             NULL, GF_PATH_COMPLETE);
-        GF_ASSERT (0 == ret);
-#endif
+        GLUSTERD_GET_VOLUME_DIR(voldir, volinfo, priv);
+
         glusterd_delete_volfile (volinfo, brickinfo);
-        glusterd_store_delete_brick (volinfo, brickinfo);
+        glusterd_store_delete_brick (brickinfo, voldir);
         glusterd_brickinfo_delete (brickinfo);
         volinfo->brick_count--;
         return ret;
@@ -4496,28 +6324,92 @@ glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo)
 }
 
 int
-mkdir_if_missing (char *path)
-{
-        struct stat st = {0,};
-        int        ret = 0;
+glusterd_get_local_brickpaths (glusterd_volinfo_t *volinfo, char **pathlist)
+{
+        char                 **path_tokens  = NULL;
+        char                  *tmp_path_list = NULL;
+        char                   path[PATH_MAX] = "";
+        int32_t                count          = 0;
+        int32_t                pathlen        = 0;
+        int32_t                total_len      = 0;
+        int32_t                ret            = 0;
+        int                    i              = 0;
+        glusterd_brickinfo_t  *brickinfo      = NULL;
+
+        if ((!volinfo) || (!pathlist))
+            goto out;
+
+        path_tokens = GF_CALLOC (sizeof(char*), volinfo->brick_count,
+                                 gf_gld_mt_charptr);
+        if (!path_tokens) {
+                gf_log ("", GF_LOG_DEBUG, "Could not allocate memory.");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
 
-        ret = mkdir (path, 0777);
-        if (!ret || errno == EEXIST)
-                ret = stat (path, &st);
-        if (ret == -1 || !S_ISDIR (st.st_mode))
-                gf_log ("", GF_LOG_WARNING, "Failed to create the"
-                        " directory %s", path);
+                 pathlen = snprintf (path, sizeof(path),
+                                     "--path=%s ", brickinfo->path);
+                 if (pathlen < sizeof(path))
+                        path[pathlen] = '\0';
+                 else
+                        path[sizeof(path)-1] = '\0';
+                 path_tokens[count] = gf_strdup (path);
+                 if (!path_tokens[count]) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not allocate memory.");
+                        ret = -1;
+                        goto out;
+                 }
+                 count++;
+                 total_len += pathlen;
+        }
+
+        tmp_path_list = GF_CALLOC (sizeof(char), total_len + 1,
+                                   gf_gld_mt_char);
+        if (!tmp_path_list) {
+                gf_log ("", GF_LOG_DEBUG, "Could not allocate memory.");
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; i < count; i++)
+                strcat (tmp_path_list, path_tokens[i]);
+
+        if (count)
+                *pathlist = tmp_path_list;
+
+        ret = count;
+out:
+        for (i = 0; i < count; i++) {
+                GF_FREE (path_tokens[i]);
+                path_tokens[i] = NULL;
+        }
+
+        GF_FREE (path_tokens);
+        path_tokens = NULL;
+
+        if (ret == 0) {
+                gf_log ("", GF_LOG_DEBUG, "No Local Bricks Present.");
+                GF_FREE (tmp_path_list);
+                tmp_path_list = NULL;
+        }
 
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
 int
 glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
-                      char *glusterd_uuid_str, char **op_errstr)
+                      char *path_list, char *conf_path,
+                      char *glusterd_uuid_str,
+                      char **op_errstr)
 {
         int32_t         ret     = 0;
         int32_t         status  = 0;
-        char            buf[PATH_MAX]   = {0,};
         char            uuid_str [64] = {0};
         runner_t        runner = {0,};
         xlator_t        *this = NULL;
@@ -4529,48 +6421,46 @@ glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
         priv = this->private;
         GF_ASSERT (priv);
 
-        uuid_utoa_r (priv->uuid, uuid_str);
-        if (strcmp (uuid_str, glusterd_uuid_str))
-                goto out;
-
-        ret = gsync_status (master_vol->volname, slave, &status);
-        if (status == 0)
-                goto out;
+        uuid_utoa_r (MY_UUID, uuid_str);
 
-        snprintf (buf, PATH_MAX, "%s/"GEOREP"/%s", priv->workdir, master_vol->volname);
-        ret = mkdir_if_missing (buf);
-        if (ret) {
-                errcode = -1;
+        if (!path_list) {
+                ret = 0;
+                gf_log ("", GF_LOG_DEBUG, "No Bricks in this node."
+                        " Not starting gsyncd.");
                 goto out;
         }
 
-        snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s",
-                  master_vol->volname);
-        ret = mkdir_if_missing (buf);
-        if (ret) {
-                errcode = -1;
+        ret = gsync_status (master_vol->volname, slave, conf_path, &status);
+        if (status == 0)
                 goto out;
-        }
 
         uuid_utoa_r (master_vol->volume_id, uuid_str);
         runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, priv->workdir);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          path_list, "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
         runner_argprintf (&runner, ":%s", master_vol->volname);
         runner_add_args  (&runner, slave, "--config-set", "session-owner",
                           uuid_str, NULL);
+        synclock_unlock (&priv->big_lock);
         ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret == -1) {
                 errcode = -1;
                 goto out;
         }
 
         runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "--monitor", "-c", NULL);
-        runner_argprintf (&runner, "%s/"GSYNC_CONF, priv->workdir);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          path_list, "--monitor", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
         runner_argprintf (&runner, ":%s", master_vol->volname);
+        runner_argprintf (&runner, "--glusterd-uuid=%s",
+                          uuid_utoa (priv->uuid));
         runner_add_arg   (&runner, slave);
+        synclock_unlock (&priv->big_lock);
         ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret == -1) {
                 gf_asprintf (op_errstr, GEOREP" start failed for %s %s",
                              master_vol->volname, slave);
@@ -4582,7 +6472,7 @@ glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
 out:
         if ((ret != 0) && errcode == -1) {
                 if (op_errstr)
-                        *op_errstr = gf_strdup ("internal error, cannot start"
+                        *op_errstr = gf_strdup ("internal error, cannot start "
                                                 "the " GEOREP " session");
         }
 
@@ -4591,17 +6481,38 @@ out:
 }
 
 int32_t
-glusterd_recreate_bricks (glusterd_conf_t *conf)
+glusterd_recreate_volfiles (glusterd_conf_t *conf)
 {
 
         glusterd_volinfo_t      *volinfo = NULL;
         int                      ret = 0;
+        int                      op_ret = 0;
 
         GF_ASSERT (conf);
         list_for_each_entry (volinfo, &conf->volumes, vol_list) {
                 ret = generate_brick_volfiles (volinfo);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate brick volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate trusted client volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate client volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
         }
-        return ret;
+        return op_ret;
 }
 
 int32_t
@@ -4611,7 +6522,8 @@ glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf)
         char            *type                           = NULL;
         gf_boolean_t     upgrade                        = _gf_false;
         gf_boolean_t     downgrade                      = _gf_false;
-        gf_boolean_t     regenerate_brick_volfiles      = _gf_false;
+        gf_boolean_t     regenerate_volfiles            = _gf_false;
+        gf_boolean_t     terminate                      = _gf_false;
 
         ret = dict_get_str (options, "upgrade", &type);
         if (!ret) {
@@ -4623,7 +6535,7 @@ glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf)
                         goto out;
                 }
                 if (_gf_true == upgrade)
-                        regenerate_brick_volfiles = _gf_true;
+                        regenerate_volfiles = _gf_true;
         }
 
         ret = dict_get_str (options, "downgrade", &type);
@@ -4646,10 +6558,14 @@ glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf)
 
         if (!upgrade && !downgrade)
                 ret = 0;
-        if (regenerate_brick_volfiles) {
-                ret = glusterd_recreate_bricks (conf);
+        else
+                terminate = _gf_true;
+        if (regenerate_volfiles) {
+                ret = glusterd_recreate_volfiles (conf);
         }
 out:
+        if (terminate && (ret == 0))
+                kill (getpid(), SIGTERM);
         return ret;
 }
 
@@ -4686,7 +6602,7 @@ glusterd_set_dump_options (char *dumpoptions_path, char *options,
                 goto out;
         }
         dup_options = gf_strdup (options);
-        gf_log ("", GF_LOG_INFO, "Recieved following statedump options: %s",
+        gf_log ("", GF_LOG_INFO, "Received following statedump options: %s",
                 dup_options);
         option = strtok_r (dup_options, " ", &tmpptr);
         while (option) {
@@ -4707,8 +6623,7 @@ glusterd_set_dump_options (char *dumpoptions_path, char *options,
 out:
         if (fp)
                 fclose (fp);
-        if (dup_options)
-                GF_FREE (dup_options);
+        GF_FREE (dup_options);
         return ret;
 }
 
@@ -4721,7 +6636,6 @@ glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
         xlator_t                *this = NULL;
         glusterd_conf_t         *conf = NULL;
         char                    pidfile_path[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
         char                    dumpoptions_path[PATH_MAX] = {0,};
         FILE                    *pidfile = NULL;
         pid_t                   pid = -1;
@@ -4741,14 +6655,12 @@ glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
                 }
         }
 
-        if (uuid_compare (brickinfo->uuid, conf->uuid)) {
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
                 ret = 0;
                 goto out;
         }
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile_path, path, brickinfo->hostname,
-                                    brickinfo->path);
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile_path, volinfo, brickinfo, conf);
 
         pidfile = fopen (pidfile_path, "r");
         if (!pidfile) {
@@ -4766,7 +6678,7 @@ glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
         }
 
         snprintf (dumpoptions_path, sizeof (dumpoptions_path),
-                  "/tmp/glusterdump.%d.options", pid);
+                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
         ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
         if (ret < 0) {
                 gf_log ("", GF_LOG_ERROR, "error while parsing the statedump "
@@ -4839,7 +6751,7 @@ glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr)
         }
 
         snprintf (dumpoptions_path, sizeof (dumpoptions_path),
-                  "/tmp/glusterdump.%d.options", pid);
+                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
         ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
         if (ret < 0) {
                 gf_log ("", GF_LOG_ERROR, "error while parsing the statedump "
@@ -4860,8 +6772,83 @@ out:
         if (pidfile)
                 fclose (pidfile);
         unlink (dumpoptions_path);
-        if (dup_options)
-                GF_FREE (dup_options);
+        GF_FREE (dup_options);
+        return ret;
+}
+
+int
+glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr)
+{
+        int                     ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *conf = NULL;
+        char                    pidfile_path[PATH_MAX] = {0,};
+        char                    path[PATH_MAX] = {0,};
+        FILE                    *pidfile = NULL;
+        pid_t                   pid = -1;
+        char                    dumpoptions_path[PATH_MAX] = {0,};
+        char                    *option = NULL;
+        char                    *tmpptr = NULL;
+        char                    *dup_options = NULL;
+        char                    msg[256] = {0,};
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        dup_options = gf_strdup (options);
+        option = strtok_r (dup_options, " ", &tmpptr);
+        if (strcmp (option, "quotad")) {
+                snprintf (msg, sizeof (msg), "for quotad statedump, options "
+                          "should be after the key 'quotad'");
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        GLUSTERD_GET_QUOTAD_DIR (path, conf);
+        GLUSTERD_GET_QUOTAD_PIDFILE (pidfile_path, path);
+
+        pidfile = fopen (pidfile_path, "r");
+        if (!pidfile) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to open pidfile: %s",
+                        pidfile_path);
+                ret = -1;
+                goto out;
+        }
+
+        ret = fscanf (pidfile, "%d", &pid);
+        if (ret <= 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get pid of quotad "
+                        "process");
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (dumpoptions_path, sizeof (dumpoptions_path),
+                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
+        ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "error while parsing "
+                        "statedump options");
+                ret = -1;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Performing statedump on quotad with "
+                "pid %d", pid);
+
+        kill (pid, SIGUSR1);
+
+        sleep (1);
+
+        ret = 0;
+out:
+        if (pidfile)
+                fclose (pidfile);
+        unlink (dumpoptions_path);
+        GF_FREE (dup_options);
         return ret;
 }
 
@@ -4996,20 +6983,44 @@ int
 glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
                               size_t len, int cmd, defrag_cbk_fn_t cbk)
 {
-        glusterd_conf_t         *priv                   = NULL;
-        char                     pidfile[PATH_MAX];
-        int                      ret                    = -1;
-        pid_t                    pid;
+        xlator_t        *this             = NULL;
+        glusterd_conf_t *priv             = NULL;
+        char            pidfile[PATH_MAX] = {0,};
+        int             ret               = -1;
+        pid_t           pid               = 0;
 
-        priv = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
         if (!priv)
                 return ret;
 
-        GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
-
-        if (!glusterd_is_service_running (pidfile, &pid)) {
+        /* Don't start the rebalance process if the stautus is already
+         * completed, stopped or failed. If the status is started, check if
+         * there is an existing process already and connect to it. If not, then
+         * start the rebalance process
+         */
+        switch (volinfo->rebal.defrag_status) {
+        case GF_DEFRAG_STATUS_COMPLETE:
+        case GF_DEFRAG_STATUS_STOPPED:
+        case GF_DEFRAG_STATUS_FAILED:
+                break;
+        case GF_DEFRAG_STATUS_STARTED:
+                GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+                if (gf_is_service_running (pidfile, &pid)) {
+                        glusterd_rebalance_rpc_create (volinfo, _gf_true);
+                        break;
+                }
+        case GF_DEFRAG_STATUS_NOT_STARTED:
                 glusterd_handle_defrag_start (volinfo, op_errstr, len, cmd,
-                                              cbk);
+                                              cbk, volinfo->rebal.op);
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unknown defrag status (%d)."
+                        "Not starting rebalance process for %s.",
+                        volinfo->rebal.defrag_status, volinfo->volname);
+                break;
         }
 
         return ret;
@@ -5023,10 +7034,2443 @@ glusterd_restart_rebalance (glusterd_conf_t *conf)
         char                     op_errstr[256];
 
         list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                if (!volinfo->defrag_cmd)
+                if (!volinfo->rebal.defrag_cmd)
+                        continue;
+                if (!gd_should_i_start_rebalance (volinfo))
                         continue;
                 glusterd_volume_defrag_restart (volinfo, op_errstr, 256,
-                                                volinfo->defrag_cmd, NULL);
+                                        volinfo->rebal.defrag_cmd, NULL);
+        }
+        return ret;
+}
+
+void
+glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo)
+{
+        glusterd_rebalance_t *rebal = NULL;
+        GF_ASSERT (volinfo);
+
+        rebal = &volinfo->rebal;
+        rebal->rebalance_files = 0;
+        rebal->rebalance_data = 0;
+        rebal->lookedup_files = 0;
+        rebal->rebalance_failures = 0;
+        rebal->rebalance_time = 0;
+        rebal->skipped_files = 0;
+
+}
+
+/* Return hostname for given uuid if it exists
+ * else return NULL
+ */
+char *
+glusterd_uuid_to_hostname (uuid_t uuid)
+{
+        char                    *hostname = NULL;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_peerinfo_t     *entry = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        if (!uuid_compare (MY_UUID, uuid)) {
+                hostname = gf_strdup ("localhost");
+        }
+        if (!list_empty (&priv->peers)) {
+                list_for_each_entry (entry, &priv->peers, uuid_list) {
+                        if (!uuid_compare (entry->uuid, uuid)) {
+                                hostname = gf_strdup (entry->hostname);
+                                break;
+                        }
+                }
+        }
+
+        return hostname;
+}
+
+gf_boolean_t
+glusterd_is_local_brick (xlator_t *this, glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo)
+{
+        gf_boolean_t    local = _gf_false;
+        int             ret = 0;
+        glusterd_conf_t *conf = NULL;
+
+        if (uuid_is_null (brickinfo->uuid)) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret)
+                        goto out;
+        }
+        conf = this->private;
+        local = !uuid_compare (brickinfo->uuid, MY_UUID);
+out:
+        return local;
+}
+int
+glusterd_validate_volume_id (dict_t *op_dict, glusterd_volinfo_t *volinfo)
+{
+        int     ret             = -1;
+        char    *volid_str      = NULL;
+        uuid_t  vol_uid         = {0, };
+        xlator_t *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (op_dict, "vol-id", &volid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume id for "
+                        "volume %s", volinfo->volname);
+                goto out;
+        }
+        ret = uuid_parse (volid_str, vol_uid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to parse volume id "
+                        "for volume %s", volinfo->volname);
+                goto out;
+        }
+
+        if (uuid_compare (vol_uid, volinfo->volume_id)) {
+                gf_log (this->name, GF_LOG_ERROR, "Volume ids of volume %s - %s"
+                        " and %s - are different. Possibly a split brain among "
+                        "peers.", volinfo->volname, volid_str,
+                        uuid_utoa (volinfo->volume_id));
+                ret = -1;
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
+                                      dict_t *rsp_dict)
+{
+        int                             ret = 0;
+        uint64_t                        files = 0;
+        uint64_t                        size = 0;
+        uint64_t                        lookup = 0;
+        gf_defrag_status_t              status = GF_DEFRAG_STATUS_NOT_STARTED;
+        uint64_t                        failures = 0;
+        uint64_t                        skipped = 0;
+        xlator_t                       *this = NULL;
+        double                          run_time = 0;
+
+        this = THIS;
+
+        ret = dict_get_uint64 (rsp_dict, "files", &files);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get file count");
+
+        ret = dict_get_uint64 (rsp_dict, "size", &size);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get size of xfer");
+
+        ret = dict_get_uint64 (rsp_dict, "lookups", &lookup);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get lookedup file count");
+
+        ret = dict_get_int32 (rsp_dict, "status", (int32_t *)&status);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get status");
+
+        ret = dict_get_uint64 (rsp_dict, "failures", &failures);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get failure count");
+
+        ret = dict_get_uint64 (rsp_dict, "skipped", &skipped);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get skipped count");
+
+        ret = dict_get_double (rsp_dict, "run-time", &run_time);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get run-time");
+
+        if (files)
+                volinfo->rebal.rebalance_files = files;
+        if (size)
+                volinfo->rebal.rebalance_data = size;
+        if (lookup)
+                volinfo->rebal.lookedup_files = lookup;
+        if (status)
+                volinfo->rebal.defrag_status = status;
+        if (failures)
+                volinfo->rebal.rebalance_failures = failures;
+        if (skipped)
+                volinfo->rebal.skipped_files = skipped;
+        if (run_time)
+                volinfo->rebal.rebalance_time = run_time;
+
+        return ret;
+}
+
+int
+glusterd_check_topology_identical (const char   *filename1,
+                                   const char   *filename2,
+                                   gf_boolean_t *identical)
+{
+        int                     ret    = -1; /* FAILURE */
+        xlator_t                *this  = THIS;
+        FILE                    *fp1   = NULL;
+        FILE                    *fp2   = NULL;
+        glusterfs_graph_t       *grph1 = NULL;
+        glusterfs_graph_t       *grph2 = NULL;
+
+        /* Invalid xlator, Nothing to do */
+        if (!this)
+                return (-1);
+
+        /* Sanitize the inputs */
+        GF_VALIDATE_OR_GOTO (this->name, filename1, out);
+        GF_VALIDATE_OR_GOTO (this->name, filename2, out);
+        GF_VALIDATE_OR_GOTO (this->name, identical, out);
+
+        /* fopen() the volfile1 to create the graph */
+        fp1 = fopen (filename1, "r");
+        if (fp1 == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
+                        "(%s)", filename1, strerror (errno));
+                goto out;
+        }
+
+        /* fopen() the volfile2 to create the graph */
+        fp2 = fopen (filename2, "r");
+        if (fp2 == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
+                        "(%s)", filename2, strerror (errno));
+                goto out;
+        }
+
+        /* create the graph for filename1 */
+        grph1 = glusterfs_graph_construct(fp1);
+        if (grph1 == NULL)
+                goto out;
+
+        /* create the graph for filename2 */
+        grph2 = glusterfs_graph_construct(fp2);
+        if (grph2 == NULL)
+                goto out;
+
+        /* compare the graph topology */
+        *identical = is_graph_topology_equal(grph1, grph2);
+        ret = 0; /* SUCCESS */
+out:
+        if (fp1)
+                fclose(fp1);
+        if (fp2)
+                fclose(fp2);
+        if (grph1)
+                glusterfs_graph_destroy(grph1);
+        if (grph2)
+                glusterfs_graph_destroy(grph2);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+glusterd_check_files_identical (char *filename1, char *filename2,
+                                gf_boolean_t *identical)
+{
+        int                     ret = -1;
+        struct stat             buf1 = {0,};
+        struct stat             buf2 = {0,};
+        uint32_t                cksum1 = 0;
+        uint32_t                cksum2 = 0;
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (filename1);
+        GF_ASSERT (filename2);
+        GF_ASSERT (identical);
+
+        this = THIS;
+
+        ret = stat (filename1, &buf1);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "stat on file: %s failed "
+                        "(%s)", filename1, strerror (errno));
+                goto out;
+        }
+
+        ret = stat (filename2, &buf2);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "stat on file: %s failed "
+                        "(%s)", filename2, strerror (errno));
+                goto out;
+        }
+
+        if (buf1.st_size != buf2.st_size) {
+                *identical = _gf_false;
+                goto out;
+        }
+
+        ret = get_checksum_for_path (filename1, &cksum1);
+        if (ret)
+                goto out;
+
+
+        ret = get_checksum_for_path (filename2, &cksum2);
+        if (ret)
+                goto out;
+
+        if (cksum1 != cksum2)
+                *identical = _gf_false;
+        else
+                *identical = _gf_true;
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+glusterd_volset_help (dict_t *dict, char **op_errstr)
+{
+        int                     ret = -1;
+        gf_boolean_t            xml_out = _gf_false;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+
+        if (!dict) {
+                if (!(dict = glusterd_op_get_ctx ())) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        if (dict_get (dict, "help" )) {
+                xml_out = _gf_false;
+
+        } else if (dict_get (dict, "help-xml" )) {
+                xml_out = _gf_true;
+#if (HAVE_LIB_XML)
+                ret = 0;
+#else
+                gf_log (this->name, GF_LOG_ERROR,
+                        "libxml not present in the system");
+                if (op_errstr)
+                        *op_errstr = gf_strdup ("Error: xml libraries not "
+                                                "present to produce "
+                                                "xml-output");
+                goto out;
+#endif
+
+        } else {
+                goto out;
         }
+
+        ret = glusterd_get_volopt_content (dict, xml_out);
+        if (ret && op_errstr)
+                *op_errstr = gf_strdup ("Failed to get volume options help");
+ out:
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+
+int
+glusterd_to_cli (rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
+                 int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
+                 dict_t *dict)
+{
+        int                ret = -1;
+        char               *cmd = NULL;
+        int                op_ret = 0;
+        char               *op_errstr = NULL;
+        int                op_errno = 0;
+        xlator_t           *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        op_ret = arg->op_ret;
+        op_errstr = arg->op_errstr;
+        op_errno = arg->op_errno;
+
+        ret = dict_get_str (dict, "cmd-str", &cmd);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get command "
+                        "string");
+
+        if (cmd) {
+                if (op_ret)
+                        gf_cmd_log ("", "%s : FAILED %s %s", cmd,
+                                       (op_errstr)? ":" : " ",
+                                       (op_errstr)? op_errstr : " ");
+                else
+                        gf_cmd_log ("", "%s : SUCCESS", cmd);
+        }
+
+        glusterd_submit_reply (req, arg, payload, payloadcount, iobref,
+                               (xdrproc_t) xdrproc);
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+static int32_t
+glusterd_append_gsync_status (dict_t *dst, dict_t *src)
+{
+        int                ret = 0;
+        char               *stop_msg = NULL;
+
+        ret = dict_get_str (src, "gsync-status", &stop_msg);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dst, "gsync-status", gf_strdup (stop_msg));
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_WARNING, "Unable to set the stop"
+                        "message in the ctx dictionary");
+                goto out;
+        }
+
+        ret = 0;
+ out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+int32_t
+glusterd_append_status_dicts (dict_t *dst, dict_t *src)
+{
+        char                sts_val_name[PATH_MAX] = {0, };
+        int                 dst_count              = 0;
+        int                 src_count              = 0;
+        int                 i                      = 0;
+        int                 ret                    = 0;
+        gf_gsync_status_t  *sts_val                = NULL;
+        gf_gsync_status_t  *dst_sts_val            = NULL;
+
+        GF_ASSERT (dst);
+
+        if (src == NULL)
+                goto out;
+
+        ret = dict_get_int32 (dst, "gsync-count", &dst_count);
+        if (ret)
+                dst_count = 0;
+
+        ret = dict_get_int32 (src, "gsync-count", &src_count);
+        if (ret || !src_count) {
+                gf_log ("", GF_LOG_DEBUG, "Source brick empty");
+                ret = 0;
+                goto out;
+        }
+
+        for (i = 0; i < src_count; i++) {
+                memset (sts_val_name, '\0', sizeof(sts_val_name));
+                snprintf (sts_val_name, sizeof(sts_val_name), "status_value%d", i);
+
+                ret = dict_get_bin (src, sts_val_name, (void **) &sts_val);
+                if (ret)
+                        goto out;
+
+                dst_sts_val = GF_CALLOC (1, sizeof(gf_gsync_status_t),
+                                         gf_common_mt_gsync_status_t);
+                if (!dst_sts_val) {
+                        gf_log ("", GF_LOG_ERROR, "Out Of Memory");
+                        goto out;
+                }
+
+                memcpy (dst_sts_val, sts_val, sizeof(gf_gsync_status_t));
+
+                memset (sts_val_name, '\0', sizeof(sts_val_name));
+                snprintf (sts_val_name, sizeof(sts_val_name), "status_value%d", i + dst_count);
+
+                ret = dict_set_bin (dst, sts_val_name, dst_sts_val, sizeof(gf_gsync_status_t));
+                if (ret)
+                        goto out;
+        }
+
+        ret = dict_set_int32 (dst, "gsync-count", dst_count+src_count);
+
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+int32_t
+glusterd_gsync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict, char *op_errstr)
+{
+        dict_t             *ctx = NULL;
+        int                ret = 0;
+        char               *conf_path = NULL;
+
+        if (aggr) {
+                ctx = aggr;
+
+        } else {
+                ctx = glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Operation Context is not present");
+                        GF_ASSERT (0);
+                }
+        }
+
+        if (rsp_dict) {
+                ret = glusterd_append_status_dicts (ctx, rsp_dict);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_append_gsync_status (ctx, rsp_dict);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (rsp_dict, "conf_path", &conf_path);
+                if (!ret && conf_path) {
+                        ret = dict_set_dynstr (ctx, "conf_path",
+                                            gf_strdup(conf_path));
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to store conf path.");
+                                goto out;
+                        }
+                }
+        }
+        if ((op_errstr) && (strcmp ("", op_errstr))) {
+                ret = dict_set_dynstr (ctx, "errstr", gf_strdup(op_errstr));
+                if (ret)
+                        goto out;
+        }
+
+        ret = 0;
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
+        return ret;
+}
+
+int32_t
+glusterd_rb_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int32_t  src_port = 0;
+        int32_t  dst_port = 0;
+        int      ret      = 0;
+        dict_t  *ctx      = NULL;
+
+
+        if (aggr) {
+                ctx = aggr;
+
+        } else {
+                ctx = glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Operation Context is not present");
+                        GF_ASSERT (0);
+                }
+        }
+
+        if (rsp_dict) {
+                ret = dict_get_int32 (rsp_dict, "src-brick-port", &src_port);
+                if (ret == 0) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "src-brick-port=%d found", src_port);
+                }
+
+                ret = dict_get_int32 (rsp_dict, "dst-brick-port", &dst_port);
+                if (ret == 0) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "dst-brick-port=%d found", dst_port);
+                }
+
+        }
+
+        if (src_port) {
+                ret = dict_set_int32 (ctx, "src-brick-port",
+                                      src_port);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not set src-brick");
+                        goto out;
+                }
+        }
+
+        if (dst_port) {
+                ret = dict_set_int32 (ctx, "dst-brick-port",
+                                      dst_port);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not set dst-brick");
+                        goto out;
+                }
+
+        }
+
+out:
+        return ret;
+
+}
+
+int32_t
+glusterd_sync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int      ret      = 0;
+
+        GF_ASSERT (rsp_dict);
+
+        if (!rsp_dict) {
+                goto out;
+        }
+
+        ret = glusterd_import_friend_volumes (rsp_dict);
+out:
+        return ret;
+
+}
+
+static int
+_profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
+                               void *data)
+{
+        char    new_key[256] = {0};
+        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
+        data_t  *new_value = NULL;
+        int     brick_count = 0;
+        char    brick_key[256];
+
+        if (strcmp (key, "count") == 0)
+                return 0;
+        sscanf (key, "%d%s", &brick_count, brick_key);
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+        snprintf (new_key, sizeof (new_key), "%d%s",
+                  rsp_ctx->count + brick_count, brick_key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+        return 0;
+}
+
+int
+glusterd_profile_volume_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int     ret = 0;
+        glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
+        int32_t brick_count = 0;
+        int32_t count = 0;
+        dict_t  *ctx_dict = NULL;
+        glusterd_op_t   op = GD_OP_NONE;
+
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (rsp_dict, "count", &brick_count);
+        if (ret) {
+                ret = 0; //no bricks in the rsp
+                goto out;
+        }
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (GD_OP_PROFILE_VOLUME == op);
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx ();
+        }
+
+        ret = dict_get_int32 (ctx_dict, "count", &count);
+        rsp_ctx.count = count;
+        rsp_ctx.dict = ctx_dict;
+        dict_foreach (rsp_dict, _profile_volume_add_friend_rsp, &rsp_ctx);
+        dict_del (ctx_dict, "count");
+        ret = dict_set_int32 (ctx_dict, "count", count + brick_count);
+out:
+        return ret;
+}
+
+static int
+glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
+                                     void *data)
+{
+        glusterd_status_rsp_conv_t      *rsp_ctx = NULL;
+        data_t                          *new_value = NULL;
+        char                            brick_key[1024] = {0,};
+        char                            new_key[1024] = {0,};
+        int32_t                         index = 0;
+        int32_t                         ret = 0;
+
+        /* Skip the following keys, they are already present in the ctx_dict */
+        /* Also, skip all the task related pairs. They will be added to the
+         * ctx_dict later
+         */
+        if (!strcmp (key, "count") || !strcmp (key, "cmd") ||
+            !strcmp (key, "brick-index-max") || !strcmp (key, "other-count") ||
+            !strncmp (key, "task", 4))
+                return 0;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+
+        sscanf (key, "brick%d.%s", &index, brick_key);
+
+        if (index > rsp_ctx->brick_index_max) {
+                snprintf (new_key, sizeof (new_key), "brick%d.%s",
+                          index + rsp_ctx->other_count, brick_key);
+        } else {
+                strncpy (new_key, key, sizeof (new_key));
+                new_key[sizeof (new_key) - 1] = 0;
+        }
+
+        ret = dict_set (rsp_ctx->dict, new_key, new_value);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Unable to set key: %s in dict",
+                        key);
+
+        return 0;
+}
+
+static int
+glusterd_volume_status_copy_tasks_to_ctx_dict (dict_t *this, char *key,
+                                               data_t *value, void *data)
+{
+        int     ret = 0;
+        dict_t  *ctx_dict = NULL;
+        data_t  *new_value = NULL;
+
+        if (strncmp (key, "task", 4))
+                return 0;
+
+        ctx_dict = data;
+        GF_ASSERT (ctx_dict);
+
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+
+        ret = dict_set (ctx_dict, key, new_value);
+
+        return ret;
+}
+
+int
+glusterd_volume_status_aggregate_tasks_status (dict_t *ctx_dict,
+                                               dict_t *rsp_dict)
+{
+        int             ret             = -1;
+        xlator_t        *this           = NULL;
+        int             local_count     = 0;
+        int             remote_count    = 0;
+        int             i               = 0;
+        int             j               = 0;
+        char            key[128]        = {0,};
+        char            *task_type      = NULL;
+        int             local_status    = 0;
+        int             remote_status   = 0;
+        char            *local_task_id  = NULL;
+        char            *remote_task_id = NULL;
+
+        GF_ASSERT (ctx_dict);
+        GF_ASSERT (rsp_dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (rsp_dict, "tasks", &remote_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get remote task count");
+                goto out;
+        }
+        /* Local count will not be present when this is called for the first
+         * time with the origins rsp_dict
+         */
+        ret = dict_get_int32 (ctx_dict, "tasks", &local_count);
+        if (ret) {
+                ret = dict_foreach (rsp_dict,
+                                glusterd_volume_status_copy_tasks_to_ctx_dict,
+                                ctx_dict);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to copy tasks"
+                                "to ctx_dict.");
+                goto out;
+        }
+
+        if (local_count != remote_count) {
+                gf_log (this->name, GF_LOG_ERROR, "Local tasks count (%d) and "
+                        "remote tasks count (%d) do not match. Not aggregating "
+                        "tasks status.", local_count, remote_count);
+                ret = -1;
+                goto out;
+        }
+
+        /* Update the tasks statuses. For every remote tasks, search for the
+         * local task, and update the local task status based on the remote
+         * status.
+         */
+        for (i = 0; i < remote_count; i++) {
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.type", i);
+                ret = dict_get_str (rsp_dict, key, &task_type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get task typpe from rsp dict");
+                        goto out;
+                }
+
+                /* Skip replace-brick status as it is going to be the same on
+                 * all peers. rb_status is set by the replace brick commit
+                 * function on all peers based on the replace brick command.
+                 * We return the value of rb_status as the status for a
+                 * replace-brick task in a 'volume status' command.
+                 */
+                if (!strcmp (task_type, "Replace brick"))
+                        continue;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "task%d.status", i);
+                ret = dict_get_int32 (rsp_dict, key, &remote_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get task status from rsp dict");
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "task%d.id", i);
+                ret = dict_get_str (rsp_dict, key, &remote_task_id);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get task id from rsp dict");
+                        goto out;
+                }
+                for (j = 0; j < local_count; j++) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "task%d.id", j);
+                        ret = dict_get_str (ctx_dict, key, &local_task_id);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get local task-id");
+                                goto out;
+                        }
+
+                        if (strncmp (remote_task_id, local_task_id,
+                                     strlen (remote_task_id))) {
+                                /* Quit if a matching local task is not found */
+                                if (j == (local_count - 1)) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Could not find matching local "
+                                                "task for task %s",
+                                                remote_task_id);
+                                        goto out;
+                                }
+                                continue;
+                        }
+
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "task%d.status", j);
+                        ret = dict_get_int32 (ctx_dict, key, &local_status);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get local task status");
+                                goto out;
+                        }
+
+                        /* Rebalance has 5 states,
+                         * NOT_STARTED, STARTED, STOPPED, COMPLETE, FAILED
+                         * The precedence used to determine the aggregate status
+                         * is as below,
+                         * STARTED > FAILED > STOPPED > COMPLETE > NOT_STARTED
+                         */
+                        /* TODO: Move this to a common place utilities that both
+                         * CLI and glusterd need.
+                         * Till then if the below algorithm is changed, change
+                         * it in cli_xml_output_vol_rebalance_status in
+                         * cli-xml-output.c
+                         */
+                        ret = 0;
+                        int rank[] = {
+                                [GF_DEFRAG_STATUS_STARTED] = 1,
+                                [GF_DEFRAG_STATUS_FAILED] = 2,
+                                [GF_DEFRAG_STATUS_STOPPED] = 3,
+                                [GF_DEFRAG_STATUS_COMPLETE] = 4,
+                                [GF_DEFRAG_STATUS_NOT_STARTED] = 5
+                        };
+                        if (rank[remote_status] <= rank[local_status])
+                                        ret = dict_set_int32 (ctx_dict, key,
+                                                              remote_status);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "update task status");
+                                goto out;
+                        }
+                        break;
+                }
+        }
+
+out:
+        return ret;
+}
+
+gf_boolean_t
+glusterd_status_has_tasks (int cmd) {
+        if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
+             (cmd & GF_CLI_STATUS_VOL))
+                return _gf_true;
+        return _gf_false;
+}
+
+int
+glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int                             ret = 0;
+        glusterd_status_rsp_conv_t      rsp_ctx = {0};
+        int32_t                         cmd = GF_CLI_STATUS_NONE;
+        int32_t                         node_count = 0;
+        int32_t                         other_count = 0;
+        int32_t                         brick_index_max = -1;
+        int32_t                         rsp_node_count = 0;
+        int32_t                         rsp_other_count = 0;
+        int                             vol_count = -1;
+        int                             i = 0;
+        dict_t                          *ctx_dict = NULL;
+        char                            key[PATH_MAX] = {0,};
+        char                            *volname = NULL;
+
+        GF_ASSERT (rsp_dict);
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (GD_OP_STATUS_VOLUME);
+
+        }
+
+        ret = dict_get_int32 (ctx_dict, "cmd", &cmd);
+        if (ret)
+                goto out;
+
+        if (cmd & GF_CLI_STATUS_ALL && is_origin_glusterd (ctx_dict)) {
+                ret = dict_get_int32 (rsp_dict, "vol_count", &vol_count);
+                if (ret == 0) {
+                        ret = dict_set_int32 (ctx_dict, "vol_count",
+                                              vol_count);
+                        if (ret)
+                                goto out;
+
+                        for (i = 0; i < vol_count; i++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "vol%d", i);
+                                ret = dict_get_str (rsp_dict, key, &volname);
+                                if (ret)
+                                        goto out;
+
+                                ret = dict_set_str (ctx_dict, key, volname);
+                                if (ret)
+                                        goto out;
+                        }
+                }
+        }
+
+        if ((cmd & GF_CLI_STATUS_TASKS) != 0)
+                goto aggregate_tasks;
+
+        ret = dict_get_int32 (rsp_dict, "count", &rsp_node_count);
+        if (ret) {
+                ret = 0; //no bricks in the rsp
+                goto out;
+        }
+
+        ret = dict_get_int32 (rsp_dict, "other-count", &rsp_other_count);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to get other count from rsp_dict");
+                goto out;
+        }
+
+        ret = dict_get_int32 (ctx_dict, "count", &node_count);
+        ret = dict_get_int32 (ctx_dict, "other-count", &other_count);
+        if (!dict_get (ctx_dict, "brick-index-max")) {
+                ret = dict_get_int32 (rsp_dict, "brick-index-max", &brick_index_max);
+                if (ret)
+                        goto out;
+                ret = dict_set_int32 (ctx_dict, "brick-index-max", brick_index_max);
+                if (ret)
+                        goto out;
+
+        } else {
+                ret = dict_get_int32 (ctx_dict, "brick-index-max", &brick_index_max);
+        }
+
+        rsp_ctx.count = node_count;
+        rsp_ctx.brick_index_max = brick_index_max;
+        rsp_ctx.other_count = other_count;
+        rsp_ctx.dict = ctx_dict;
+
+        dict_foreach (rsp_dict, glusterd_volume_status_add_peer_rsp, &rsp_ctx);
+
+        ret = dict_set_int32 (ctx_dict, "count", node_count + rsp_node_count);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to update node count");
+                goto out;
+        }
+
+        ret = dict_set_int32 (ctx_dict, "other-count",
+                              (other_count + rsp_other_count));
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to update other-count");
+                goto out;
+        }
+
+aggregate_tasks:
+        /* Tasks are only present for a normal status command for a volume or
+         * for an explicit tasks status command for a volume
+         */
+        if (!(cmd & GF_CLI_STATUS_ALL) &&
+            (((cmd & GF_CLI_STATUS_TASKS) != 0) ||
+             glusterd_status_has_tasks (cmd)))
+                ret = glusterd_volume_status_aggregate_tasks_status (ctx_dict,
+                                                                     rsp_dict);
+
+out:
+        return ret;
+}
+
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        char                 key[256]      = {0,};
+        char                *node_uuid     = NULL;
+        char                *node_uuid_str = NULL;
+        char                *volname       = NULL;
+        dict_t              *ctx_dict      = NULL;
+        double               elapsed_time  = 0;
+        glusterd_conf_t     *conf          = NULL;
+        glusterd_op_t        op            = GD_OP_NONE;
+        glusterd_peerinfo_t *peerinfo      = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        int32_t              index         = 0;
+        int32_t              count         = 0;
+        int32_t              current_index = 2;
+        int32_t              value32       = 0;
+        uint64_t             value         = 0;
+        char                *peer_uuid_str = NULL;
+
+        GF_ASSERT (rsp_dict);
+        conf = THIS->private;
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT ((GD_OP_REBALANCE == op) ||
+                   (GD_OP_DEFRAG_BRICK_VOLUME == op));
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (op);
+
+        }
+
+        if (!ctx_dict)
+                goto out;
+
+        ret = dict_get_str (ctx_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (rsp_dict, "count", &index);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "failed to get index");
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "node-uuid-%d", index);
+        ret = dict_get_str (rsp_dict, key, &node_uuid);
+        if (!ret) {
+                node_uuid_str = gf_strdup (node_uuid);
+
+                /* Finding the index of the node-uuid in the peer-list */
+                list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
+                        peer_uuid_str = gd_peer_uuid_str (peerinfo);
+                        if (strcmp (peer_uuid_str, node_uuid_str) == 0)
+                                break;
+
+                        current_index++;
+                }
+
+                /* Setting the largest index value as the total count. */
+                ret = dict_get_int32 (ctx_dict, "count", &count);
+                if (count < current_index) {
+                    ret = dict_set_int32 (ctx_dict, "count", current_index);
+                    if (ret)
+                            gf_log ("", GF_LOG_ERROR, "Failed to set count");
+                }
+
+                /* Setting the same index for the node, as is in the peerlist.*/
+                memset (key, 0, 256);
+                snprintf (key, 256, "node-uuid-%d", current_index);
+                ret = dict_set_dynstr (ctx_dict, key, node_uuid_str);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set node-uuid");
+                }
+        }
+
+        snprintf (key, 256, "files-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "files-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set the file count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "size-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "size-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set the size of migration");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "lookups-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "lookups-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set lookuped file count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "status-%d", index);
+        ret = dict_get_int32 (rsp_dict, key, &value32);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", current_index);
+                ret = dict_set_int32 (ctx_dict, key, value32);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set status");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "failures-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "failures-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set failure count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "skipped-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "skipped-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set skipped count");
+                }
+        }
+        memset (key, 0, 256);
+        snprintf (key, 256, "run-time-%d", index);
+        ret = dict_get_double (rsp_dict, key, &elapsed_time);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", current_index);
+                ret = dict_set_double (ctx_dict, key, elapsed_time);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set run-time");
+                }
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_sys_exec_output_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char           output_name[PATH_MAX] = "";
+        char          *output = NULL;
+        int            ret      = 0;
+        int            i      = 0;
+        int            len    = 0;
+        int            src_output_count      = 0;
+        int            dst_output_count      = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "output_count", &dst_output_count);
+
+        ret = dict_get_int32 (src, "output_count", &src_output_count);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "No output from source");
+                ret = 0;
+                goto out;
+        }
+
+        for (i = 1; i <= src_output_count; i++) {
+                len = snprintf (output_name, sizeof(output_name) - 1,
+                                "output_%d", i);
+                output_name[len] = '\0';
+                ret = dict_get_str (src, output_name, &output);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch %s",
+                                output_name);
+                        goto out;
+                }
+
+                len = snprintf (output_name, sizeof(output_name) - 1,
+                                "output_%d", i+dst_output_count);
+                output_name[len] = '\0';
+                ret = dict_set_dynstr (dst, output_name, gf_strdup (output));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to set %s",
+                                output_name);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dst, "output_count",
+                              dst_output_count+src_output_count);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int            ret      = 0;
+        glusterd_op_t  op       = GD_OP_NONE;
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (aggr);
+        GF_ASSERT (rsp_dict);
+
+        if (!aggr)
+                goto out;
+        dict_copy (rsp_dict, aggr);
+out:
+        return ret;
+}
+
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int            ret      = 0;
+        dict_t        *ctx_dict = NULL;
+        glusterd_op_t  op       = GD_OP_NONE;
+
+        GF_ASSERT (rsp_dict);
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (GD_OP_HEAL_VOLUME == op);
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (op);
+        }
+
+        if (!ctx_dict)
+                goto out;
+        dict_copy (rsp_dict, ctx_dict);
+out:
+        return ret;
+}
+
+int
+_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                             void *data)
+{
+        char    new_key[256] = {0};
+        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
+        data_t  *new_value = NULL;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+        snprintf (new_key, sizeof (new_key), "%d-%s", rsp_ctx->count, key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+        return 0;
+}
+
+int
+glusterd_volume_quota_copy_to_op_ctx_dict (dict_t *dict, dict_t *rsp_dict)
+{
+        int        ret            = -1;
+        int        i              = 0;
+        int        count          = 0;
+        int        rsp_dict_count = 0;
+        char      *uuid_str       = NULL;
+        char      *uuid_str_dup   = NULL;
+        char       key[256]       = {0,};
+        xlator_t  *this           = NULL;
+        int        type           = GF_QUOTA_OPTION_TYPE_NONE;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get quota opcode");
+                goto out;
+        }
+
+        if ((type != GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) &&
+            (type != GF_QUOTA_OPTION_TYPE_REMOVE)) {
+                dict_copy (rsp_dict, dict);
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_int32 (rsp_dict, "count", &rsp_dict_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get the count of "
+                        "gfids from the rsp dict");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                /* The key "count" is absent in op_ctx when this function is
+                 * called after self-staging on the originator. This must not
+                 * be treated as error.
+                 */
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to get count of gfids"
+                        " from req dict. This could be because count is not yet"
+                        " copied from rsp_dict into op_ctx");
+
+        for (i = 0; i < rsp_dict_count; i++) {
+                snprintf (key, sizeof(key)-1, "gfid%d", i);
+
+                ret = dict_get_str (rsp_dict, key, &uuid_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get gfid "
+                                "from rsp dict");
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key)-1, "gfid%d", i + count);
+
+                uuid_str_dup = gf_strdup (uuid_str);
+                if (!uuid_str_dup) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_dynstr (dict, key, uuid_str_dup);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set gfid "
+                                "from rsp dict into req dict");
+                        GF_FREE (uuid_str_dup);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "count", rsp_dict_count + count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,  "Failed to set aggregated "
+                        "count in req dict");
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_profile_volume_brick_rsp (void *pending_entry,
+                                   dict_t *rsp_dict, dict_t *op_ctx,
+                                   char **op_errstr, gd_node_type type)
+{
+        int                             ret = 0;
+        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
+        int32_t                         count = 0;
+        char                            brick[PATH_MAX+1024] = {0};
+        char                            key[256] = {0};
+        char                            *full_brick = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (pending_entry);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_int32 (op_ctx, "count", &count);
+        if (ret) {
+                count = 1;
+        } else {
+                count++;
+        }
+        snprintf (key, sizeof (key), "%d-brick", count);
+        if (type == GD_NODE_BRICK) {
+                brickinfo = pending_entry;
+                snprintf (brick, sizeof (brick), "%s:%s", brickinfo->hostname,
+                          brickinfo->path);
+        } else if (type == GD_NODE_NFS) {
+                snprintf (brick, sizeof (brick), "%s", uuid_utoa (MY_UUID));
+        }
+        full_brick = gf_strdup (brick);
+        GF_ASSERT (full_brick);
+        ret = dict_set_dynstr (op_ctx, key, full_brick);
+
+        rsp_ctx.count = count;
+        rsp_ctx.dict = op_ctx;
+        dict_foreach (rsp_dict, _profile_volume_add_brick_rsp, &rsp_ctx);
+        dict_del (op_ctx, "count");
+        ret = dict_set_int32 (op_ctx, "count", count);
+        return ret;
+}
+
+//input-key: <replica-id>:<child-id>-*
+//output-key: <brick-id>-*
+int
+_heal_volume_add_shd_rsp (dict_t *this, char *key, data_t *value, void *data)
+{
+        char                            new_key[256] = {0,};
+        char                            int_str[16] = {0};
+        data_t                          *new_value = NULL;
+        char                            *rxl_end = NULL;
+        char                            *rxl_child_end = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        int                             rxl_id = 0;
+        int                             rxl_child_id = 0;
+        int                             brick_id = 0;
+        int                             int_len = 0;
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+
+        rsp_ctx = data;
+        rxl_end = strchr (key, '-');
+        if (!rxl_end)
+                goto out;
+
+        int_len = strlen (key) - strlen (rxl_end);
+        strncpy (int_str, key, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_id);
+        if (ret)
+                goto out;
+
+        rxl_child_end = strchr (rxl_end + 1, '-');
+        if (!rxl_child_end)
+                goto out;
+
+        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
+        strncpy (int_str, rxl_end + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_child_id);
+        if (ret)
+                goto out;
+
+        volinfo = rsp_ctx->volinfo;
+        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+        if (!strcmp (rxl_child_end, "-status")) {
+                brickinfo = glusterd_get_brickinfo_by_position (volinfo,
+                                                                brick_id);
+                if (!brickinfo)
+                        goto out;
+                if (!glusterd_is_local_brick (rsp_ctx->this, volinfo,
+                                              brickinfo))
+                        goto out;
+        }
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "%d%s", brick_id, rxl_child_end);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+out:
+        return 0;
+}
+
+int
+_heal_volume_add_shd_rsp_of_statistics (dict_t *this, char *key, data_t
+                                             *value, void *data)
+{
+        char                            new_key[256] = {0,};
+        char                            int_str[16] = {0,};
+        char                            key_begin_string[128] = {0,};
+        data_t                          *new_value = NULL;
+        char                            *rxl_end = NULL;
+        char                            *rxl_child_end = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        char                            *key_begin_str = NULL;
+        int                             rxl_id = 0;
+        int                             rxl_child_id = 0;
+        int                             brick_id = 0;
+        int                             int_len = 0;
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+
+        rsp_ctx = data;
+        key_begin_str = strchr (key, '-');
+        if (!key_begin_str)
+                goto out;
+
+        int_len = strlen (key) - strlen (key_begin_str);
+        strncpy (key_begin_string, key, int_len);
+        key_begin_string[int_len] = '\0';
+
+        rxl_end = strchr (key_begin_str + 1, '-');
+        if (!rxl_end)
+                goto out;
+
+        int_len = strlen (key_begin_str) - strlen (rxl_end) - 1;
+        strncpy (int_str, key_begin_str + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_id);
+        if (ret)
+                goto out;
+
+
+        rxl_child_end = strchr (rxl_end + 1, '-');
+        if (!rxl_child_end)
+                goto out;
+
+        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
+        strncpy (int_str, rxl_end + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_child_id);
+        if (ret)
+                goto out;
+
+        volinfo = rsp_ctx->volinfo;
+        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+        brickinfo = glusterd_get_brickinfo_by_position (volinfo, brick_id);
+        if (!brickinfo)
+                goto out;
+        if (!glusterd_is_local_brick (rsp_ctx->this, volinfo, brickinfo))
+                goto out;
+
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "%s-%d%s", key_begin_string,
+                  brick_id, rxl_child_end);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+out:
+        return 0;
+
+}
+
+int
+glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                dict_t *op_ctx, char **op_errstr)
+{
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        rsp_ctx = {0};
+        char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        int                             heal_op = -1;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_str (req_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (req_dict, "heal-op", &heal_op);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get heal_op");
+                goto out;
+        }
+
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        rsp_ctx.dict = op_ctx;
+        rsp_ctx.volinfo = volinfo;
+        rsp_ctx.this = THIS;
+        if (heal_op == GF_AFR_OP_STATISTICS)
+                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp_of_statistics,
+                              &rsp_ctx);
+        else
+                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp, &rsp_ctx);
+
+
+out:
+        return ret;
+}
+
+int
+_status_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                              void *data)
+{
+        char                            new_key[256] = {0,};
+        data_t                          *new_value = 0;
+        glusterd_pr_brick_rsp_conv_t    *rsp_ctx = NULL;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "brick%d.%s", rsp_ctx->count, key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+        return 0;
+}
+
+int
+glusterd_status_volume_brick_rsp (dict_t *rsp_dict, dict_t *op_ctx,
+                                  char **op_errstr)
+{
+        int                             ret = 0;
+        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
+        int32_t                         count = 0;
+        int                             index = 0;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_int32 (op_ctx, "count", &count);
+        if (ret) {
+                count = 0;
+        } else {
+                count++;
+        }
+        ret = dict_get_int32 (rsp_dict, "index", &index);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get node index");
+                goto out;
+        }
+        dict_del (rsp_dict, "index");
+
+        rsp_ctx.count = index;
+        rsp_ctx.dict = op_ctx;
+        dict_foreach (rsp_dict, _status_volume_add_brick_rsp, &rsp_ctx);
+        ret = dict_set_int32 (op_ctx, "count", count);
+
+out:
+        return ret;
+}
+
+int
+glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                 dict_t *op_ctx)
+{
+        int                             ret = 0;
+        char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        char                            key[256] = {0,};
+        int32_t                         i = 0;
+        char                            buf[1024] = {0,};
+        char                            *node_str = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (req_dict);
+
+        ret = dict_get_str (req_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        if (rsp_dict) {
+                ret = glusterd_defrag_volume_status_update (volinfo,
+                                                            rsp_dict);
+        }
+
+        if (!op_ctx) {
+                dict_copy (rsp_dict, op_ctx);
+                goto out;
+        }
+
+        ret = dict_get_int32 (op_ctx, "count", &i);
+        i++;
+
+        ret = dict_set_int32 (op_ctx, "count", i);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set count");
+
+        snprintf (buf, 1024, "%s", uuid_utoa (MY_UUID));
+        node_str = gf_strdup (buf);
+
+        snprintf (key, 256, "node-uuid-%d",i);
+        ret = dict_set_dynstr (op_ctx, key, node_str);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set node-uuid");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "files-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set file count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "size-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_data);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set size of xfer");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "lookups-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.lookedup_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set lookedup file count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "status-%d", i);
+        ret = dict_set_int32 (op_ctx, key, volinfo->rebal.defrag_status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set status");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "failures-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set failure count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "skipped-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.skipped_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set skipped count");
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "run-time-%d", i);
+        ret = dict_set_double (op_ctx, key, volinfo->rebal.rebalance_time);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set run-time");
+
+out:
+        return ret;
+}
+int32_t
+glusterd_handle_node_rsp (dict_t *req_dict, void *pending_entry,
+                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
+                          char **op_errstr, gd_node_type type)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (op_errstr);
+
+        switch (op) {
+        case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_brick_rsp (pending_entry,
+                                                         rsp_dict, op_ctx,
+                                                         op_errstr, type);
+                break;
+        case GD_OP_STATUS_VOLUME:
+                ret = glusterd_status_volume_brick_rsp (rsp_dict, op_ctx,
+                                                        op_errstr);
+                break;
+
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                glusterd_defrag_volume_node_rsp (req_dict,
+                                                 rsp_dict, op_ctx);
+                break;
+
+        case GD_OP_HEAL_VOLUME:
+                ret = glusterd_heal_volume_brick_rsp (req_dict, rsp_dict,
+                                                      op_ctx, op_errstr);
+                break;
+        default:
+                break;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_set_originator_uuid (dict_t *dict)
+{
+        int          ret              = -1;
+        uuid_t      *originator_uuid  = NULL;
+
+        GF_ASSERT (dict);
+
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+out:
+        if (ret && originator_uuid)
+                GF_FREE (originator_uuid);
+
+        return ret;
+}
+
+/* Should be used only when an operation is in progress, as that is the only
+ * time a lock_owner is set
+ */
+gf_boolean_t
+is_origin_glusterd (dict_t *dict)
+{
+        gf_boolean_t  ret              = _gf_false;
+        uuid_t        lock_owner       = {0,};
+        uuid_t        *originator_uuid = NULL;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_bin (dict, "originator_uuid",
+                            (void **) &originator_uuid);
+        if (ret) {
+                /* If not originator_uuid has been set, then the command
+                 * has been originated from a glusterd running on older version
+                 * Hence fetching the lock owner */
+                ret = glusterd_get_lock_owner (&lock_owner);
+                if (ret) {
+                        ret = _gf_false;
+                        goto out;
+                }
+                ret = !uuid_compare (MY_UUID, lock_owner);
+        } else
+                ret = !uuid_compare (MY_UUID, *originator_uuid);
+
+out:
+        return ret;
+}
+
+int
+glusterd_generate_and_set_task_id (dict_t *dict, char *key)
+{
+        int             ret = -1;
+        uuid_t          task_id = {0,};
+        char            *uuid_str = NULL;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        uuid_generate (task_id);
+        uuid_str = gf_strdup (uuid_utoa (task_id));
+        if (!uuid_str) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dict, key, uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s in dict",
+                        key);
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_INFO, "Generated task-id %s for key %s",
+                uuid_str, key);
+
+out:
+        if (ret)
+                GF_FREE (uuid_str);
+        return ret;
+}
+
+int
+glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key)
+{
+        int             ret = -1;
+        char            tmp_str[40] = {0,};
+        char            *task_id_str = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+
+        uuid_unparse (uuid, tmp_str);
+        task_id_str = gf_strdup (tmp_str);
+        if (!task_id_str)
+                return -1;
+
+        ret = dict_set_dynstr (dict, key, task_id_str);
+        if (ret) {
+                GF_FREE (task_id_str);
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Error setting uuid in dict with key %s", key);
+        }
+
+        return 0;
+}
+
+int
+_update_volume_op_versions (dict_t *this, char *key, data_t *value, void *data)
+{
+        int                op_version = 0;
+        glusterd_volinfo_t *ctx       = NULL;
+        gf_boolean_t       enabled    = _gf_true;
+        int                ret        = -1;
+
+        GF_ASSERT (data);
+        ctx = data;
+
+        op_version = glusterd_get_op_version_for_key (key);
+
+        if (gd_is_xlator_option (key) || gd_is_boolean_option (key)) {
+                ret = gf_string2boolean (value->data, &enabled);
+                if (ret)
+                        return 0;
+
+                if (!enabled)
+                        return 0;
+        }
+
+        if (op_version > ctx->op_version)
+                ctx->op_version = op_version;
+
+        if (gd_is_client_option (key) &&
+            (op_version > ctx->client_op_version))
+                ctx->client_op_version = op_version;
+
+        return 0;
+}
+
+void
+gd_update_volume_op_versions (glusterd_volinfo_t *volinfo)
+{
+        glusterd_conf_t *conf = NULL;
+        gf_boolean_t    ob_enabled = _gf_false;
+
+        GF_ASSERT (volinfo);
+
+        conf = THIS->private;
+        GF_ASSERT (conf);
+
+        /* Reset op-versions to minimum */
+        volinfo->op_version = 1;
+        volinfo->client_op_version = 1;
+
+        dict_foreach (volinfo->dict, _update_volume_op_versions, volinfo);
+
+        /* Special case for open-behind
+         * If cluster op-version >= 2 and open-behind hasn't been explicitly
+         * disabled, volume op-versions must be updated to account for it
+         */
+
+        /* TODO: Remove once we have a general way to update automatically
+         * enabled features
+         */
+        if (conf->op_version >= 2) {
+                ob_enabled = dict_get_str_boolean (volinfo->dict,
+                                                   "performance.open-behind",
+                                                   _gf_true);
+                if (ob_enabled) {
+
+                        if (volinfo->op_version < 2)
+                                volinfo->op_version = 2;
+                        if (volinfo->client_op_version < 2)
+                                volinfo->client_op_version = 2;
+                }
+        }
+
+        return;
+}
+
+int
+op_version_check (xlator_t *this, int min_op_version, char *msg, int msglen)
+{
+       int              ret  = 0;
+       glusterd_conf_t *priv = NULL;
+
+       GF_ASSERT (this);
+       GF_ASSERT (msg);
+
+       priv = this->private;
+       if (priv->op_version < min_op_version) {
+                snprintf (msg, msglen, "One or more nodes do not support "
+                          "the required op-version. Cluster op-version must "
+                          "atleast be %d.", min_op_version);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+      }
+      return ret;
+}
+
+
+/* A task is committed/completed once the task-id for it is cleared */
+gf_boolean_t
+gd_is_remove_brick_committed (glusterd_volinfo_t *volinfo)
+{
+        GF_ASSERT (volinfo);
+
+        if ((GD_OP_REMOVE_BRICK == volinfo->rebal.op) &&
+            !uuid_is_null (volinfo->rebal.rebalance_id))
+                        return _gf_false;
+
+        return _gf_true;
+}
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr)
+{
+        glusterd_peerinfo_t   *peerinfo  = NULL;
+        glusterd_brickinfo_t  *brickinfo = NULL;
+        gf_boolean_t           ret       = _gf_false;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (!uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                list_for_each_entry (peerinfo, peers, uuid_list) {
+                        if (uuid_compare (peerinfo->uuid, brickinfo->uuid))
+                                continue;
+
+                        /*Found peer who owns the brick, return false
+                         * if peer is not connected or not friend */
+                        if (!(peerinfo->connected) ||
+                           (peerinfo->state.state !=
+                             GD_FRIEND_STATE_BEFRIENDED)) {
+                                *down_peerstr = gf_strdup (peerinfo->hostname);
+                                gf_log ("", GF_LOG_DEBUG, "Peer %s is down. ",
+                                        peerinfo->hostname);
+                                goto out;
+                        }
+                }
+       }
+
+        ret = _gf_true;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict)
+{
+        int           ret             = -1;
+        uint32_t      cmd             = GF_CLI_STATUS_NONE;
+        gf_boolean_t  is_status_tasks = _gf_false;
+
+        if (op != GD_OP_STATUS_VOLUME)
+                goto out;
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get opcode");
+                goto out;
+        }
+
+        if (cmd & GF_CLI_STATUS_TASKS)
+                is_status_tasks = _gf_true;
+
+out:
+        return is_status_tasks;
+}
+
+/* Tells if rebalance needs to be started for the given volume on the peer
+ *
+ * Rebalance should be started on a peer only if an involved brick is present on
+ * the peer.
+ *
+ * For a normal rebalance, if any one brick of the given volume is present on
+ * the peer, the rebalance process should be started.
+ *
+ * For a rebalance as part of a remove-brick operation, the rebalance process
+ * should be started only if one of the bricks being removed is present on the
+ * peer
+ */
+gf_boolean_t
+gd_should_i_start_rebalance  (glusterd_volinfo_t *volinfo) {
+        gf_boolean_t         retval     = _gf_false;
+        int                  ret        = -1;
+        glusterd_brickinfo_t *brick     = NULL;
+        int                  count      = 0;
+        int                  i          = 0;
+        char                 key[1023]  = {0,};
+        char                 *brickname = NULL;
+
+
+        switch (volinfo->rebal.op) {
+        case GD_OP_REBALANCE:
+                list_for_each_entry (brick, &volinfo->bricks, brick_list) {
+                        if (uuid_compare (MY_UUID, brick->uuid) == 0) {
+                                retval = _gf_true;
+                                break;
+                        }
+                }
+                break;
+        case GD_OP_REMOVE_BRICK:
+                ret = dict_get_int32 (volinfo->rebal.dict, "count", &count);
+                if (ret) {
+                        goto out;
+                }
+                for (i = 1; i <= count; i++) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "brick%d", i);
+                        ret = dict_get_str (volinfo->rebal.dict, key,
+                                            &brickname);
+                        if (ret)
+                                goto out;
+                        ret = glusterd_volume_brickinfo_get_by_brick (brickname,
+                                                                      volinfo,
+                                                                      &brick);
+                        if (ret)
+                                goto out;
+                        if (uuid_compare (MY_UUID, brick->uuid) == 0) {
+                                retval = _gf_true;
+                                break;
+                        }
+                }
+                break;
+        default:
+                break;
+        }
+
+out:
+        return retval;
+}
+
+int
+glusterd_is_volume_quota_enabled (glusterd_volinfo_t *volinfo)
+{
+        return (glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA));
+}
+
+int
+glusterd_validate_and_set_gfid (dict_t *op_ctx, dict_t *req_dict,
+                                char **op_errstr)
+{
+        int        ret           = -1;
+        int        count         = 0;
+        int        i             = 0;
+        int        op_code       = GF_QUOTA_OPTION_TYPE_NONE;
+        uuid_t     uuid1         = {0};
+        uuid_t     uuid2         = {0,};
+        char      *path          = NULL;
+        char       key[256]      = {0,};
+        char      *uuid1_str     = NULL;
+        char      *uuid1_str_dup = NULL;
+        char      *uuid2_str     = NULL;
+        xlator_t  *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (op_ctx, "type", &op_code);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get quota opcode");
+                goto out;
+        }
+
+        if ((op_code != GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) &&
+            (op_code != GF_QUOTA_OPTION_TYPE_REMOVE)) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_str (op_ctx, "path", &path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get path");
+                goto out;
+        }
+
+        ret = dict_get_int32 (op_ctx, "count", &count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get count");
+                goto out;
+        }
+
+        /* If count is 0, fail the command with ENOENT.
+         *
+         * If count is 1, treat gfid0 as the gfid on which the operation
+         * is to be performed and resume the command.
+         *
+         * if count > 1, get the 0th gfid from the op_ctx and,
+         * compare it with the remaining 'count -1' gfids.
+         * If they are found to be the same, set gfid0 in the op_ctx and
+         * resume the operation, else error out.
+         */
+
+        if (count == 0) {
+                gf_asprintf (op_errstr, "Failed to get trusted.gfid attribute "
+                             "on path %s. Reason : %s", path,
+                             strerror (ENOENT));
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key) - 1, "gfid%d", 0);
+
+        ret = dict_get_str (op_ctx, key, &uuid1_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get key '%s'",
+                        key);
+                goto out;
+        }
+
+        uuid_parse (uuid1_str, uuid1);
+
+        for (i = 1; i < count; i++) {
+                snprintf (key, sizeof (key)-1, "gfid%d", i);
+
+                ret = dict_get_str (op_ctx, key, &uuid2_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get key "
+                                "'%s'", key);
+                        goto out;
+                }
+
+                uuid_parse (uuid2_str, uuid2);
+
+                if (uuid_compare (uuid1, uuid2)) {
+                        gf_asprintf (op_errstr, "gfid mismatch between %s and "
+                                     "%s for path %s", uuid1_str, uuid2_str,
+                                     path);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (i == count) {
+                uuid1_str_dup = gf_strdup (uuid1_str);
+                if (!uuid1_str_dup) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_dynstr (req_dict, "gfid", uuid1_str_dup);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set gfid");
+                        GF_FREE (uuid1_str_dup);
+                        goto out;
+                }
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to iterate through %d"
+                        " entries in the req dict", count);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+glusterd_clean_up_quota_store (glusterd_volinfo_t *volinfo)
+{
+        char      voldir[PATH_MAX]         = {0,};
+        char      quota_confpath[PATH_MAX] = {0,};
+        char      cksum_path[PATH_MAX]     = {0,};
+        xlator_t  *this                    = NULL;
+        glusterd_conf_t *conf              = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, conf);
+
+        snprintf (quota_confpath, sizeof (quota_confpath), "%s/%s", voldir,
+                  GLUSTERD_VOLUME_QUOTA_CONFIG);
+        snprintf (cksum_path, sizeof (cksum_path), "%s/%s", voldir,
+                  GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+
+        unlink (quota_confpath);
+        unlink (cksum_path);
+
+        gf_store_handle_destroy (volinfo->quota_conf_shandle);
+        volinfo->quota_conf_shandle = NULL;
+        volinfo->quota_conf_version = 0;
+
+}
+
+#define QUOTA_CONF_HEADER                                                \
+        "GlusterFS Quota conf | version: v%d.%d\n"
+
+int
+glusterd_store_quota_conf_skip_header (xlator_t *this, int fd)
+{
+        char buf[PATH_MAX] = {0,};
+
+        snprintf (buf, sizeof(buf)-1, QUOTA_CONF_HEADER, 1, 1);
+        return gf_skip_header_section (fd, strlen (buf));
+}
+
+int
+glusterd_store_quota_conf_stamp_header (xlator_t *this, int fd)
+{
+        char buf[PATH_MAX]  = {0,};
+        int  buf_len        = 0;
+        ssize_t  ret        = -1;
+        ssize_t  written    = 0;
+
+        snprintf (buf, sizeof(buf)-1, QUOTA_CONF_HEADER, 1, 1);
+        buf_len = strlen (buf);
+        for (written = 0; written != buf_len; written += ret) {
+                ret = write (fd, buf + written, buf_len - written);
+                if (ret == -1) {
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_remove_auxiliary_mount (char *volname)
+{
+        int       ret                = -1;
+        runner_t  runner             = {0,};
+        char      mountdir[PATH_MAX] = {0,};
+        char      pidfile[PATH_MAX]  = {0,};
+        xlator_t *this               = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile, volname);
+
+        if (!gf_is_service_running (pidfile, NULL)) {
+                gf_log (this->name, GF_LOG_DEBUG, "Aux mount of volume %s "
+                        "absent, hence returning", volname);
+                return 0;
+        }
+
+        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
+        runinit (&runner);
+        runner_add_args (&runner, "umount",
+
+#if GF_LINUX_HOST_OS
+                        "-l",
+#endif
+                        mountdir, NULL);
+        ret = runner_run_reuse (&runner);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "umount on %s failed, "
+                        "reason : %s", mountdir, strerror (errno));
+        runner_end (&runner);
+
+        rmdir (mountdir);
+        return ret;
+}
+
+/* Stops the rebalance process of the given volume
+ */
+int
+gd_stop_rebalance_process (glusterd_volinfo_t *volinfo)
+{
+        int              ret               = -1;
+        xlator_t        *this              = NULL;
+        glusterd_conf_t *conf              = NULL;
+        char             pidfile[PATH_MAX] = {0,};
+
+        GF_ASSERT (volinfo);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GLUSTERD_GET_DEFRAG_PID_FILE (pidfile, volinfo, conf);
+        ret = glusterd_service_stop ("rebalance", pidfile, SIGTERM, _gf_true);
+
+        return ret;
+}
+
+rpc_clnt_t *
+glusterd_rpc_clnt_unref (glusterd_conf_t *conf, rpc_clnt_t *rpc)
+{
+        rpc_clnt_t *ret = NULL;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (rpc);
+        synclock_unlock (&conf->big_lock);
+        ret = rpc_clnt_unref (rpc);
+        synclock_lock (&conf->big_lock);
+
+        return ret;
+}
+
+int32_t
+glusterd_compare_volume_name(struct list_head *list1, struct list_head *list2)
+{
+        glusterd_volinfo_t *volinfo1 = NULL;
+        glusterd_volinfo_t *volinfo2 = NULL;
+
+        volinfo1 = list_entry(list1, glusterd_volinfo_t, vol_list);
+        volinfo2 = list_entry(list2, glusterd_volinfo_t, vol_list);
+        return strcmp(volinfo1->volname, volinfo2->volname);
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index de6185753..ec59d9143 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_UTILS_H
 #define _GLUSTERD_UTILS_H_
 
@@ -38,24 +28,27 @@
 #include "rpc-clnt.h"
 #include "protocol-common.h"
 
-/* For brick search using path: partial or complete */
-typedef enum {
-        GF_PATH_COMPLETE = 0,
-        GF_PATH_PARTIAL
-} gf_path_match_t;
+#define GLUSTERD_SOCK_DIR "/var/run"
+#define GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid) do {\
+        sprintf (brickinfo->brick_id, "%s-client-%d",\
+                 volinfo->volname, brickid);\
+} while (0)
 
 struct glusterd_lock_ {
         uuid_t  owner;
         time_t  timestamp;
 };
 
-typedef struct glusterd_voldict_ctx_ {
+typedef struct glusterd_dict_ctx_ {
         dict_t  *dict;
-        int     count;
         int     opt_count;
         char    *key_name;
         char    *val_name;
-} glusterd_voldict_ctx_t;
+        char    *prefix;
+} glusterd_dict_ctx_t;
+
+int
+glusterd_compare_lines (const void *a, const void *b);
 
 typedef int (*glusterd_condition_func) (glusterd_volinfo_t *volinfo,
                                         glusterd_brickinfo_t *brickinfo,
@@ -80,6 +73,11 @@ glusterd_submit_reply (rpcsvc_request_t *req, void *arg,
                        struct iobref *iobref, xdrproc_t xdrproc);
 
 int
+glusterd_to_cli (rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
+                 int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
+                 dict_t *dict);
+
+int
 glusterd_submit_request (struct rpc_clnt *rpc, void *req,
                          call_frame_t *frame, rpc_clnt_prog_t *prog,
                          int procnum, struct iobref *iobref,
@@ -109,7 +107,7 @@ int32_t
 glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo);
 
 int32_t
-glusterd_brickinfo_from_brick (char *brick, glusterd_brickinfo_t **brickinfo);
+glusterd_brickinfo_new_from_brick (char *brick, glusterd_brickinfo_t **brickinfo);
 
 int32_t
 glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo);
@@ -123,20 +121,34 @@ glusterd_peer_hostname_new (char *hostname, glusterd_peer_hostname_t **name);
 int32_t
 glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo);
 
+int
+glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo);
+
 int32_t
 glusterd_service_stop(const char *service, char *pidfile, int sig,
                       gf_boolean_t force_kill);
 
+int
+glusterd_get_next_available_brickid (glusterd_volinfo_t *volinfo);
+
 int32_t
 glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo);
 
 int32_t
 glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
-                                 glusterd_brickinfo_t   *brickinfo);
+                                 glusterd_brickinfo_t   *brickinfo,
+                                 gf_boolean_t wait);
 
 int32_t
 glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
-                                glusterd_brickinfo_t   *brickinfo);
+                                glusterd_brickinfo_t   *brickinfo,
+                                gf_boolean_t del_brick);
+
+glusterd_volinfo_t *
+glusterd_volinfo_ref (glusterd_volinfo_t *volinfo);
+
+glusterd_volinfo_t *
+glusterd_volinfo_unref (glusterd_volinfo_t *volinfo);
 
 int32_t
 glusterd_volinfo_delete (glusterd_volinfo_t *volinfo);
@@ -150,28 +162,23 @@ glusterd_is_cli_op_req (int32_t op);
 int32_t
 glusterd_volume_brickinfo_get_by_brick (char *brick,
                                         glusterd_volinfo_t *volinfo,
-                                        glusterd_brickinfo_t **brickinfo,
-                                        gf_path_match_t path_match);
-int32_t
-glusterd_is_local_addr (char *hostname);
+                                        glusterd_brickinfo_t **brickinfo);
 
 int32_t
 glusterd_build_volume_dict (dict_t **vols);
 
 int32_t
-glusterd_compare_friend_data (dict_t  *vols, int32_t *status);
+glusterd_compare_friend_data (dict_t  *vols, int32_t *status, char *hostname);
 
 int
-glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo);
+glusterd_compute_cksum (glusterd_volinfo_t  *volinfo,
+                        gf_boolean_t is_quota_conf);
 
 void
 glusterd_get_nodesvc_volfile (char *server, char *workdir,
                                     char *volfile, size_t len);
 
 gf_boolean_t
-glusterd_is_service_running (char *pidfile, int *pid);
-
-gf_boolean_t
 glusterd_is_nodesvc_running ();
 
 gf_boolean_t
@@ -193,26 +200,35 @@ int32_t
 glusterd_shd_stop ();
 
 int32_t
-glusterd_shd_set_socket_filepath (char *rundir, uuid_t uuid,
-                                  char *socketpath, int len);
+glusterd_quotad_start ();
+
+int32_t
+glusterd_quotad_stop ();
+
+void
+glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len);
+
+int32_t
+glusterd_nodesvc_set_socket_filepath (char *rundir, uuid_t uuid,
+                                      char *socketpath, int len);
 
 struct rpc_clnt*
 glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node);
 
 struct rpc_clnt*
-glusterd_shd_get_rpc (void);
+glusterd_nodesvc_get_rpc (char *server);
 
 int32_t
-glusterd_shd_set_rpc (struct rpc_clnt *rpc);
+glusterd_nodesvc_set_rpc (char *server, struct rpc_clnt *rpc);
 
 int32_t
-glusterd_shd_connect (char *socketpath);
+glusterd_nodesvc_connect (char *server, char *socketpath);
 
 void
-glusterd_shd_set_running (gf_boolean_t status);
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status);
 
 gf_boolean_t
-glusterd_shd_is_running ();
+glusterd_is_nodesvc_online (char *server);
 
 int
 glusterd_remote_hostname_get (rpcsvc_request_t *req,
@@ -229,6 +245,9 @@ int
 glusterd_check_generate_start_shd (void);
 
 int
+glusterd_check_generate_start_quotad (void);
+
+int
 glusterd_nodesvcs_handle_graph_change (glusterd_volinfo_t *volinfo);
 
 int
@@ -246,8 +265,8 @@ int32_t
 glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                              dict_t  *dict, int32_t count);
 int
-glusterd_get_brickinfo (xlator_t *this, const char *brickname, 
-                        int port, gf_boolean_t localhost, 
+glusterd_get_brickinfo (xlator_t *this, const char *brickname,
+                        int port, gf_boolean_t localhost,
                         glusterd_brickinfo_t **brickinfo);
 
 void
@@ -271,10 +290,12 @@ glusterd_all_volume_cond_check (glusterd_condition_func func, int status,
                                 void *ctx);
 int
 glusterd_brick_start (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo);
+                      glusterd_brickinfo_t *brickinfo,
+                      gf_boolean_t wait);
 int
 glusterd_brick_stop (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo);
+                     glusterd_brickinfo_t *brickinfo,
+                     gf_boolean_t del_brick);
 
 int
 glusterd_is_defrag_on (glusterd_volinfo_t *volinfo);
@@ -293,8 +314,8 @@ glusterd_volume_brickinfos_delete (glusterd_volinfo_t *volinfo);
 int32_t
 glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
                                glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t **brickinfo,
-                               gf_path_match_t path_match);
+                               glusterd_brickinfo_t **brickinfo);
+
 int
 glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path,
                        glusterd_brickinfo_t **brickinfo);
@@ -312,19 +333,25 @@ glusterd_is_rb_ongoing (glusterd_volinfo_t *volinfo);
 
 int
 glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
-                          glusterd_brickinfo_t *src_brick, glusterd_brickinfo_t *dst_brick);
+                          glusterd_brickinfo_t *src_brick,
+                          glusterd_brickinfo_t *dst_brick);
+
+int
+glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
+                                    char **op_errstr, gf_boolean_t is_force);
 
 int
-glusterd_brick_create_path (char *host, char *path, uuid_t uuid, mode_t mode,
-                            char **op_errstr);
+glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
+                                        uuid_t volume_id, char **op_errstr,
+                                        gf_boolean_t is_force);
 int
 glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
                                            int old_state, int new_state,
                                            int event);
 int
 glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
-                       glusterd_friend_sm_state_t state,
-                       uuid_t *uuid, const char *hostname);
+                       glusterd_friend_sm_state_t state, uuid_t *uuid,
+                       const char *hostname, int port);
 int
 glusterd_sm_tr_log_init (glusterd_sm_tr_log_t *log,
                          char * (*state_name_get) (int),
@@ -355,10 +382,18 @@ glusterd_delete_brick (glusterd_volinfo_t* volinfo,
 int32_t
 glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo);
 int
+glusterd_spawn_daemons (void *opaque);
+int
 glusterd_restart_gsyncds (glusterd_conf_t *conf);
 int
 glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
-                      char *glusterd_uuid_str, char **op_errstr);
+                      char *path_list, char *conf_path,
+                      char *glusterd_uuid_str,
+                      char **op_errstr);
+int
+glusterd_get_local_brickpaths (glusterd_volinfo_t *volinfo,
+                               char **pathlist);
+
 int32_t
 glusterd_recreate_bricks (glusterd_conf_t *conf);
 int32_t
@@ -386,8 +421,13 @@ glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
                           char *options, int option_cnt, char **op_errstr);
 int
 glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr);
+
+int
+glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr);
+
 gf_boolean_t
 glusterd_is_volume_replicate (glusterd_volinfo_t *volinfo);
+
 gf_boolean_t
 glusterd_is_brick_decommissioned (glusterd_volinfo_t *volinfo, char *hostname,
                                   char *path);
@@ -412,5 +452,212 @@ int
 glusterd_restart_rebalance (glusterd_conf_t *conf);
 
 int32_t
-glusterd_add_bricks_hname_path_to_dict (dict_t *dict);
+glusterd_add_bricks_hname_path_to_dict (dict_t *dict,
+                                        glusterd_volinfo_t *volinfo);
+
+int
+glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
+                           dict_t *vol_opts);
+
+char *
+glusterd_uuid_to_hostname (uuid_t uuid);
+
+int
+glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo);
+
+glusterd_brickinfo_t*
+glusterd_get_brickinfo_by_position (glusterd_volinfo_t *volinfo, uint32_t pos);
+
+gf_boolean_t
+glusterd_is_local_brick (xlator_t *this, glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo);
+int
+glusterd_validate_volume_id (dict_t *op_dict, glusterd_volinfo_t *volinfo);
+
+int
+glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
+                                      dict_t *rsp_dict);
+
+int
+glusterd_check_files_identical (char *filename1, char *filename2,
+                                gf_boolean_t *identical);
+
+int
+glusterd_check_topology_identical (const char *filename1,
+                                   const char *filename2,
+                                   gf_boolean_t *identical);
+
+void
+glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo);
+int
+glusterd_volset_help (dict_t *dict, char **op_errstr);
+
+int32_t
+glusterd_sync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int32_t
+glusterd_gsync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict, char *op_errstr);
+int32_t
+glusterd_rb_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_profile_volume_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_sys_exec_output_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int32_t
+glusterd_handle_node_rsp (dict_t *req_ctx, void *pending_entry,
+                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
+                          char **op_errstr, gd_node_type type);
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+
+int32_t
+glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo);
+int
+glusterd_volume_quota_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp);
+int
+_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                             void *data);
+int
+glusterd_profile_volume_brick_rsp (void *pending_entry,
+                                   dict_t *rsp_dict, dict_t *op_ctx,
+                                   char **op_errstr, gd_node_type type);
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr);
+
+int32_t
+glusterd_set_originator_uuid (dict_t *dict);
+
+/* Should be used only when an operation is in progress, as that is the only
+ * time a lock_owner is set
+ */
+gf_boolean_t
+is_origin_glusterd (dict_t *dict);
+
+gf_boolean_t
+glusterd_is_quorum_changed (dict_t *options, char *option, char *value);
+
+int
+glusterd_do_quorum_action ();
+
+int
+glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
+                                    int *quorum_count);
+
+int
+glusterd_get_next_global_opt_version_str (dict_t *opts, char **version_str);
+gf_boolean_t
+glusterd_is_quorum_option (char *option);
+gf_boolean_t
+glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo);
+gf_boolean_t
+glusterd_is_any_volume_in_server_quorum (xlator_t *this);
+gf_boolean_t
+does_gd_meet_server_quorum (xlator_t *this);
+
+int
+glusterd_generate_and_set_task_id (dict_t *dict, char *key);
+
+int
+glusterd_validate_and_set_gfid (dict_t *op_ctx, dict_t *req_dict,
+                                char **op_errstr);
+
+int
+glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key);
+
+gf_boolean_t
+glusterd_is_same_address (char *name1, char *name2);
+
+void
+gd_update_volume_op_versions (glusterd_volinfo_t *volinfo);
+
+int
+op_version_check (xlator_t *this, int min_op_version, char *msg, int msglen);
+
+char*
+gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo);
+
+gf_boolean_t
+gd_is_remove_brick_committed (glusterd_volinfo_t *volinfo);
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr);
+
+int
+glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char **slave_ip, char **slave_vol,
+                                     char **conf_path, char **op_errstr);
+
+int
+glusterd_get_slave_info (char *slave, char **slave_ip,
+                         char **slave_vol, char **op_errstr);
+
+int
+glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
+                             char *conf_path, char **statefile);
+
+int
+glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen);
+
+int
+glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
+                                      dict_t *resp_dict, char *path_list,
+                                      char *conf_path, gf_boolean_t is_force);
+
+int
+glusterd_check_gsync_running_local (char *master, char *slave,
+                                    char *conf_path,
+                                    gf_boolean_t *is_run);
+
+gf_boolean_t
+glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict);
+
+gf_boolean_t
+gd_should_i_start_rebalance  (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_is_volume_quota_enabled (glusterd_volinfo_t *volinfo);
+
+gf_boolean_t
+glusterd_all_volumes_with_quota_stopped ();
+
+int
+glusterd_reconfigure_quotad ();
+
+void
+glusterd_clean_up_quota_store (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_store_quota_conf_skip_header (xlator_t *this, int fd);
+
+int
+glusterd_store_quota_conf_stamp_header (xlator_t *this, int fd);
+
+int
+glusterd_remove_auxiliary_mount (char *volname);
+
+gf_boolean_t
+glusterd_status_has_tasks (int cmd);
+
+int
+gd_stop_rebalance_process (glusterd_volinfo_t *volinfo);
+
+rpc_clnt_t *
+glusterd_rpc_clnt_unref (glusterd_conf_t *conf, rpc_clnt_t *rpc);
+
+int32_t
+glusterd_compare_volume_name(struct list_head *, struct list_head *);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 4b51054fa..9012003c9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -25,6 +15,7 @@
 
 #include <fnmatch.h>
 #include <sys/wait.h>
+#include <dlfcn.h>
 
 #if (HAVE_LIB_XML)
 #include <libxml/encoding.h>
@@ -44,170 +35,9 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "run.h"
+#include "options.h"
 
-
-/* dispatch table for VOLUME SET
- * -----------------------------
- *
- * Format of entries:
- *
- * First field is the <key>, for the purpose of looking it up
- * in volume dictionary. Each <key> is of the format "<domain>.<specifier>".
- *
- * Second field is <voltype>.
- *
- * Third field is <option>, if its unset, it's assumed to be
- * the same as <specifier>.
- *
- * Fourth field is <value>. In this context they are used to specify
- * a default. That is, even the volume dict doesn't have a value,
- * we procced as if the default value were set for it.
- *
- * There are two type of entries: basic and special.
- *
- * - Basic entries are the ones where the <option> does _not_ start with
- *   the bang! character ('!').
- *
- *   In their case, <option> is understood as an option for an xlator of
- *   type <voltype>. Their effect is to copy over the volinfo->dict[<key>]
- *   value to all graph nodes of type <voltype> (if such a value is set).
- *
- *   You are free to add entries of this type, they will become functional
- *   just by being present in the table.
- *
- * - Special entries where the <option> starts with the bang!.
- *
- *   They are not applied to all graphs during generation, and you cannot
- *   extend them in a trivial way which could be just picked up. Better
- *   not touch them unless you know what you do.
- *
- * "NODOC" entries are not part of the public interface and are subject
- * to change at any time.
- *
- * Another kind of grouping for options, according to visibility:
- *
- * - Exported: one which is used in the code. These are characterized by
- *   being used a macro as <key> (of the format VKEY_..., defined in
- *   glusterd-volgen.h
- *
- * - Non-exported: the rest; these have string literal <keys>.
- *
- * Adhering to this policy, option name changes shall be one-liners.
- *
- */
-typedef enum  { DOC, NO_DOC, GLOBAL_DOC, GLOBAL_NO_DOC } option_type_t;
-
-
-struct volopt_map_entry {
-        char *key;
-        char *voltype;
-        char *option;
-        char *value;
-        option_type_t type;
-        uint32_t flags;
-};
-
-static struct volopt_map_entry glusterd_volopt_map[] = {
-
-        {"cluster.lookup-unhashed",              "cluster/distribute", NULL, NULL, NO_DOC, 0},
-        {"cluster.min-free-disk",                "cluster/distribute", NULL, NULL, NO_DOC, 0},
-	{"cluster.min-free-inodes",              "cluster/distribute", NULL, NULL, NO_DOC, 0},
-
-        {"cluster.entry-change-log",             "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.read-subvolume",               "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.background-self-heal-count",   "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.metadata-self-heal",           "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.data-self-heal",               "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.entry-self-heal",              "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.self-heal-daemon",             "cluster/replicate",  "!self-heal-daemon", NULL, NO_DOC, 0},
-        {"cluster.strict-readdir",               "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.self-heal-window-size",        "cluster/replicate",  "data-self-heal-window-size", NULL, DOC, 0},
-        {"cluster.data-change-log",              "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.metadata-change-log",          "cluster/replicate",  NULL, NULL, NO_DOC, 0},
-        {"cluster.data-self-heal-algorithm",     "cluster/replicate",  "data-self-heal-algorithm", NULL,DOC, 0},
-        {"cluster.quorum-type",                  "cluster/replicate",  "quorum-type", NULL, NO_DOC, 0},
-        {"cluster.quorum-count",                 "cluster/replicate",  "quorum-count", NULL, NO_DOC, 0},
-
-        {"cluster.stripe-block-size",            "cluster/stripe",     "block-size", NULL, DOC, 0},
-
-        {VKEY_DIAG_LAT_MEASUREMENT,              "debug/io-stats",     "latency-measurement", "off", NO_DOC, 0},
-        {"diagnostics.dump-fd-stats",            "debug/io-stats",     NULL, NULL, NO_DOC, 0},
-        {VKEY_DIAG_CNT_FOP_HITS,                 "debug/io-stats",     "count-fop-hits", "off", NO_DOC, 0},
-
-        {"diagnostics.brick-log-level",          "debug/io-stats",     "!brick-log-level", NULL, DOC, 0},
-        {"diagnostics.client-log-level",         "debug/io-stats",     "!client-log-level", NULL, DOC, 0},
-        {"diagnostics.brick-sys-log-level",      "debug/io-stats",     "!sys-log-level", NULL, DOC, 0},
-        {"diagnostics.client-sys-log-level",     "debug/io-stats",     "!sys-log-level", NULL, DOC, 0},
-
-        {"performance.cache-max-file-size",      "performance/io-cache",      "max-file-size", NULL, DOC, 0},
-        {"performance.cache-min-file-size",      "performance/io-cache",      "min-file-size", NULL, DOC, 0},
-        {"performance.cache-refresh-timeout",    "performance/io-cache",      "cache-timeout", NULL, DOC, 0},
-        {"performance.cache-priority",           "performance/io-cache",      "priority", NULL, DOC, 0},
-        {"performance.cache-size",               "performance/io-cache",      NULL, NULL, NO_DOC, 0 },
-        {"performance.cache-size",               "performance/quick-read",    NULL, NULL, NO_DOC, 0 },
-        {"performance.flush-behind",             "performance/write-behind",  "flush-behind", NULL, DOC, 0},
-
-        {"performance.io-thread-count",          "performance/io-threads",    "thread-count", DOC, 0},
-        {"performance.disk-usage-limit",         "performance/quota",         NULL, NULL, NO_DOC, 0},
-        {"performance.min-free-disk-limit",      "performance/quota",         NULL, NULL, NO_DOC, 0},
-        {"performance.write-behind-window-size", "performance/write-behind",  "cache-size", NULL, DOC},
-        {"performance.read-ahead-page-count",    "performance/read-ahead",    "page-count", NULL, DOC},
-
-        {"network.frame-timeout",                "protocol/client",           NULL, NULL, NO_DOC, 0},
-        {"network.ping-timeout",                 "protocol/client",           NULL, NULL, NO_DOC, 0},
-        {"network.tcp-window-size",              "protocol/client",           NULL, NULL, NO_DOC, 0},
-
-        {"network.tcp-window-size",              "protocol/server",           NULL, NULL, NO_DOC, 0},
-        {"network.inode-lru-limit",              "protocol/server",           NULL, NULL, NO_DOC, 0},
-        {"auth.allow",                           "protocol/server",           "!server-auth", "*", DOC, 0},
-        {"auth.reject",                          "protocol/server",           "!server-auth", NULL, DOC, 0},
-        {"transport.keepalive",                  "protocol/server",           "transport.socket.keepalive", NULL, NO_DOC, 0},
-        {"server.allow-insecure",                "protocol/server",           "rpc-auth-allow-insecure", NULL, NO_DOC, 0},
-
-        {"performance.write-behind",             "performance/write-behind",  "!perf", "on", NO_DOC, 0},
-        {"performance.read-ahead",               "performance/read-ahead",    "!perf", "on", NO_DOC, 0},
-        {"performance.io-cache",                 "performance/io-cache",      "!perf", "on", NO_DOC, 0},
-        {"performance.quick-read",               "performance/quick-read",    "!perf", "on", NO_DOC, 0},
-        {VKEY_PERF_STAT_PREFETCH,                "performance/md-cache",      "!perf", "on", NO_DOC, 0},
-        {"performance.client-io-threads",        "performance/io-threads",    "!perf", "off", NO_DOC, 0},
-        {VKEY_MARKER_XTIME,                      "features/marker",           "xtime", "off", NO_DOC, OPT_FLAG_FORCE},
-        {VKEY_MARKER_XTIME,                      "features/marker",           "!xtime", "off", NO_DOC, OPT_FLAG_FORCE},
-        {"debug.trace",                          "debug/trace",               "!debug","off", NO_DOC, 0},
-        {"debug.error-gen",                      "debug/error-gen",           "!debug","off", NO_DOC, 0},
-
-        {"nfs.enable-ino32",                     "nfs/server",                "nfs.enable-ino32", NULL, GLOBAL_DOC, 0},
-        {"nfs.mem-factor",                       "nfs/server",                "nfs.mem-factor", NULL, GLOBAL_DOC, 0},
-        {"nfs.export-dirs",                      "nfs/server",                "nfs3.export-dirs", NULL, GLOBAL_DOC, 0},
-        {"nfs.export-volumes",                   "nfs/server",                "nfs3.export-volumes", NULL, GLOBAL_DOC, 0},
-        {"nfs.addr-namelookup",                  "nfs/server",                "rpc-auth.addr.namelookup", NULL, GLOBAL_DOC, 0},
-        {"nfs.dynamic-volumes",                  "nfs/server",                "nfs.dynamic-volumes", NULL, GLOBAL_NO_DOC, 0},
-        {"nfs.register-with-portmap",            "nfs/server",                "rpc.register-with-portmap", NULL, GLOBAL_DOC, 0},
-        {"nfs.port",                             "nfs/server",                "nfs.port", NULL, GLOBAL_DOC, 0},
-
-        {"nfs.rpc-auth-unix",                    "nfs/server",                "!rpc-auth.auth-unix.*", NULL, DOC, 0},
-        {"nfs.rpc-auth-null",                    "nfs/server",                "!rpc-auth.auth.null.*", NULL, DOC, 0},
-        {"nfs.rpc-auth-allow",                   "nfs/server",                "!rpc-auth.addr.*.allow", NULL, DOC, 0},
-        {"nfs.rpc-auth-reject",                  "nfs/server",                "!rpc-auth.addr.*.reject", NULL, DOC, 0},
-        {"nfs.ports-insecure",                   "nfs/server",                "!rpc-auth.ports.*.insecure", NULL, DOC, 0},
-        {"nfs.transport-type",                   "nfs/server",                "!nfs.transport-type", NULL, DOC, 0},
-
-        {"nfs.trusted-sync",                     "nfs/server",                "!nfs3.*.trusted-sync", NULL, DOC, 0},
-        {"nfs.trusted-write",                    "nfs/server",                "!nfs3.*.trusted-write", NULL, DOC, 0},
-        {"nfs.volume-access",                    "nfs/server",                "!nfs3.*.volume-access", NULL, DOC, 0},
-        {"nfs.export-dir",                       "nfs/server",                "!nfs3.*.export-dir", NULL, DOC, 0},
-        {"nfs.disable",                          "nfs/server",                "!nfs-disable", NULL, DOC, 0},
-
-        {VKEY_FEATURES_QUOTA,                    "features/marker",           "quota", "off", NO_DOC, OPT_FLAG_FORCE},
-        {VKEY_FEATURES_LIMIT_USAGE,              "features/quota",            "limit-set", NULL, NO_DOC, 0},
-        {"features.quota-timeout",               "features/quota",            "timeout", "0", DOC, 0},
-        {"server.statedump-path",                "protocol/server",           "statedump-path", NULL, NO_DOC, 0},
-        {"client.lk-heal",                       "protocol/client",           "lk-heal", NULL, DOC, 0},
-        {"client.grace-timeout",                 "protocol/client",           "grace-timeout", NULL, DOC, 0},
-        {"server.grace-timeout",                 "protocol/server",           "grace-timeout", NULL, DOC, 0},
-        {NULL,                                                                }
-};
-
-
+extern struct volopt_map_entry glusterd_volopt_map[];
 
 /*********************************************
  *
@@ -257,13 +87,14 @@ xlator_instantiate_va (const char *type, const char *format, va_list arg)
         xl->name = volname;
         INIT_LIST_HEAD (&xl->volume_options);
 
+        xl->ctx = THIS->ctx;
+
         return xl;
 
  error:
         gf_log ("", GF_LOG_ERROR, "creating xlator of type %s failed",
                 type);
-        if (volname)
-                GF_FREE (volname);
+        GF_FREE (volname);
         if (xl)
                 xlator_destroy (xl);
 
@@ -532,10 +363,9 @@ volopt_trie_section (int lvl, char **patt, char *word, char **hint, int hints)
         GF_ASSERT (hints <= 2);
         nodevec.cnt = hints;
         ret = trie_measure_vec (trie, word, &nodevec);
-        if (ret || !nodevec.nodes[0])
-                trie_destroy (trie);
+        if (!ret && nodevec.nodes[0])
+                ret = process_nodevec (&nodevec, hint);
 
-        ret = process_nodevec (&nodevec, hint);
         trie_destroy (trie);
 
         return ret;
@@ -586,8 +416,7 @@ volopt_trie (char *key, char **hint)
         }
 
  out:
-        if (patt[0])
-                GF_FREE (patt[0]);
+        GF_FREE (patt[0]);
         if (ret)
                 *hint = NULL;
 
@@ -619,26 +448,21 @@ struct opthandler_data {
         void *param;
 };
 
-#define pattern_match_options 0
-
-
-static void
-process_option (dict_t *dict, char *key, data_t *value, void *param)
+static int
+process_option (char *key, data_t *value, void *param)
 {
         struct opthandler_data *odt = param;
         struct volopt_map_entry vme = {0,};
 
         if (odt->rv)
-                return;
-#if pattern_match_options
-        if (fnmatch (odt->vme->key, key, 0) != 0)
-                return;
-#endif
+                return 0;
         odt->found = _gf_true;
 
         vme.key = key;
         vme.voltype = odt->vme->voltype;
         vme.option = odt->vme->option;
+        vme.op_version = odt->vme->op_version;
+
         if (!vme.option) {
                 vme.option = strrchr (key, '.');
                 if (vme.option)
@@ -652,6 +476,7 @@ process_option (dict_t *dict, char *key, data_t *value, void *param)
                 vme.value = value->data;
 
         odt->rv = odt->handler (odt->graph, &vme, odt->param);
+        return 0;
 }
 
 static int
@@ -672,14 +497,10 @@ volgen_graph_set_options_generic (volgen_graph_t *graph, dict_t *dict,
                 odt.found = _gf_false;
                 odt.data_t_fake = _gf_false;
 
-#if pattern_match_options
-                dict_foreach (dict, process_option, &odt);
-#else
                 data = dict_get (dict, vme->key);
 
                 if (data)
-                        process_option (dict, vme->key, data, &odt);
-#endif
+                        process_option (vme->key, data, &odt);
                 if (odt.rv)
                         return odt.rv;
 
@@ -693,8 +514,7 @@ volgen_graph_set_options_generic (volgen_graph_t *graph, dict_t *dict,
                          * in this context
                          */
                         odt.data_t_fake = _gf_true;
-                        process_option (NULL, vme->key, (data_t *)vme->value,
-                                        &odt);
+                        process_option (vme->key, (data_t *)vme->value, &odt);
                         if (odt.rv)
                                 return odt.rv;
                 }
@@ -754,33 +574,99 @@ optget_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
         return 0;
 }
 
+static glusterd_server_xlator_t
+get_server_xlator (char *xlator)
+{
+        glusterd_server_xlator_t subvol = GF_XLATOR_NONE;
+
+        if (strcmp (xlator, "posix") == 0)
+                subvol = GF_XLATOR_POSIX;
+        if (strcmp (xlator, "acl") == 0)
+                subvol = GF_XLATOR_ACL;
+        if (strcmp (xlator, "locks") == 0)
+                subvol = GF_XLATOR_LOCKS;
+        if (strcmp (xlator, "io-threads") == 0)
+                subvol = GF_XLATOR_IOT;
+        if (strcmp (xlator, "index") == 0)
+                subvol = GF_XLATOR_INDEX;
+        if (strcmp (xlator, "marker") == 0)
+                subvol = GF_XLATOR_MARKER;
+        if (strcmp (xlator, "io-stats") == 0)
+                subvol = GF_XLATOR_IO_STATS;
+        if (strcmp (xlator, "bd") == 0)
+                subvol = GF_XLATOR_BD;
+
+        return subvol;
+}
+
+static glusterd_client_xlator_t
+get_client_xlator (char *xlator)
+{
+        glusterd_client_xlator_t subvol = GF_CLNT_XLATOR_NONE;
+
+        if (strcmp (xlator, "client") == 0)
+                subvol = GF_CLNT_XLATOR_FUSE;
+
+        return subvol;
+}
+
+static int
+debugxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
+{
+        char *volname = NULL;
+        gf_boolean_t enabled = _gf_false;
+
+        volname = param;
+
+        if (strcmp (vme->option, "!debug") != 0)
+                return 0;
+
+        if (!strcmp (vme->key , "debug.trace") ||
+            !strcmp (vme->key, "debug.error-gen")) {
+                if (get_server_xlator (vme->value) == GF_XLATOR_NONE &&
+                    get_client_xlator (vme->value) == GF_CLNT_XLATOR_NONE)
+                        return 0;
+                else
+                        goto add_graph;
+        }
+
+        if (gf_string2boolean (vme->value, &enabled) == -1)
+                return -1;
+        if (!enabled)
+                return 0;
+
+add_graph:
+        if (volgen_graph_add (graph, vme->voltype, volname))
+                return 0;
+        else
+                return -1;
+}
+
 int
 check_and_add_debug_xl (volgen_graph_t *graph, dict_t *set_dict, char *volname,
                         char *xlname)
 {
         int       ret       = 0;
-        xlator_t *xl        = NULL;
         char     *value_str = NULL;
 
         ret = dict_get_str (set_dict, "debug.trace", &value_str);
         if (!ret) {
                 if (strcmp (xlname, value_str) == 0) {
-                        xl = volgen_graph_add (graph, "debug/trace", volname);
-                        if (!xl) {
-                                ret = -1;
+                        ret = volgen_graph_set_options_generic (graph, set_dict, volname,
+                                                                &debugxl_option_handler);
+                        if (ret)
                                 goto out;
-                        }
                 }
         }
 
         ret = dict_get_str (set_dict, "debug.error-gen", &value_str);
         if (!ret) {
                 if (strcmp (xlname, value_str) == 0) {
-                        xl = volgen_graph_add (graph, "debug/error-gen", volname);
-                        if (!xl) {
-                                ret = -1;
+                        ret = volgen_graph_set_options_generic (graph, set_dict, volname,
+                                                                &debugxl_option_handler);
+                        if (ret)
                                 goto out;
-                        }
                 }
         }
 
@@ -958,20 +844,20 @@ glusterd_check_voloption (char *key)
 int
 glusterd_check_option_exists (char *key, char **completion)
 {
-        dict_t *dict = NULL;
         struct volopt_map_entry vme = {0,};
         struct volopt_map_entry *vmep = NULL;
         int ret = 0;
+        xlator_t *this = THIS;
 
         (void)vme;
         (void)vmep;
-        (void)dict;
 
         if (!strchr (key, '.')) {
                 if (completion) {
                         ret = option_complete (key, completion);
                         if (ret) {
-                                gf_log ("", GF_LOG_ERROR, "Out of memory");
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Out of memory");
                                 return -1;
                         }
 
@@ -984,42 +870,12 @@ glusterd_check_option_exists (char *key, char **completion)
                         return 0;
         }
 
-#if !pattern_match_options
         for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
                 if (strcmp (vmep->key, key) == 0) {
                         ret = 1;
                         break;
                 }
         }
-#else
-        vme.key = key;
-
-        /* We are getting a bit anal here to avoid typing
-         * fnmatch one more time. Orthogonality foremost!
-         * The internal logic of looking up in the volopt_map table
-         * should be coded exactly once.
-         *
-         * [[Ha-ha-ha, so now if I ever change the internals then I'll
-         * have to update the fnmatch in this comment also :P ]]
-         */
-        dict = get_new_dict ();
-        if (!dict || dict_set_str (dict, key, "")) {
-                gf_log ("", GF_LOG_ERROR, "Out of memory");
-
-                return -1;
-        }
-
-        ret = volgen_graph_set_options_generic (NULL, dict, &vme,
-                                                &optget_option_handler);
-        dict_destroy (dict);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Out of memory");
-
-                return -1;
-        }
-
-        ret = !!vme.value;
-#endif
 
         if (ret || !completion)
                 return ret;
@@ -1027,7 +883,7 @@ glusterd_check_option_exists (char *key, char **completion)
  trie:
         ret = volopt_trie (key, completion);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "Some error occurred during keyword hinting");
         }
 
@@ -1211,8 +1067,7 @@ volgen_write_volfile (volgen_graph_t *graph, char *filename)
 
  error:
 
-        if (ftmp)
-                GF_FREE (ftmp);
+        GF_FREE (ftmp);
         if (f)
                 fclose (f);
 
@@ -1548,25 +1403,31 @@ static int
 server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                       dict_t *set_dict, void *param)
 {
-        char     *volname                 = NULL;
-        char     *path                    = NULL;
-        int       pump                    = 0;
-        xlator_t *xl                      = NULL;
-        xlator_t *txl                     = NULL;
-        xlator_t *rbxl                    = NULL;
-        char      transt[16]              = {0,};
-        char     *ptranst                 = NULL;
-        char      volume_id[64]           = {0,};
-        char      tstamp_file[PATH_MAX]   = {0,};
-        int       ret                     = 0;
-        char     *xlator                  = NULL;
-        char     *loglevel                = NULL;
-        char     *username                = NULL;
-        char     *password                = NULL;
-        char     index_basepath[PATH_MAX] = {0};
-        char     key[1024]                = {0};
-
-        path = param;
+        char                 *volname       = NULL;
+        char                 *path          = NULL;
+        int                   pump          = 0;
+        xlator_t             *xl            = NULL;
+        xlator_t             *txl           = NULL;
+        xlator_t             *rbxl          = NULL;
+        char      transt[16]                = {0,};
+        char                 *ptranst       = NULL;
+        char      volume_id[64]             = {0,};
+        char      tstamp_file[PATH_MAX]     = {0,};
+        int                   ret           = 0;
+        char                 *xlator        = NULL;
+        char                 *loglevel      = NULL;
+        char                 *username      = NULL;
+        char                 *password      = NULL;
+        char     index_basepath[PATH_MAX]   = {0};
+        char     key[1024]                  = {0};
+        glusterd_brickinfo_t *brickinfo     = NULL;
+        char changelog_basepath[PATH_MAX]   = {0,};
+        gf_boolean_t          quota_enabled = _gf_true;
+        gf_boolean_t          pgfid_feat    = _gf_false;
+        char                 *value         = NULL;
+
+        brickinfo = param;
+        path      = brickinfo->path;
         volname = volinfo->volname;
         get_vol_transport_type (volinfo, transt);
 
@@ -1582,6 +1443,22 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 }
         }
 
+        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_QUOTA, &value);
+        if (value) {
+                ret = gf_string2boolean (value, &quota_enabled);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_volinfo_get (volinfo,
+                                    "update-link-count-parent",
+                                    &value);
+        if (value) {
+                ret = gf_string2boolean (value, &pgfid_feat);
+                if (ret)
+                        goto out;
+        }
+
         xl = volgen_graph_add (graph, "storage/posix", volname);
         if (!xl)
                 return -1;
@@ -1595,7 +1472,50 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 return -1;
 
-        ret = check_and_add_debug_xl (graph, set_dict, volname, "posix");
+        if (quota_enabled || pgfid_feat)
+                xlator_set_option (xl, "update-link-count-parent",
+                                   "on");
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "posix");
+        if (ret)
+                return -1;
+#ifdef HAVE_BD_XLATOR
+        if (*brickinfo->vg != '\0') {
+                /* Now add BD v2 xlator if volume is BD type */
+                xl = volgen_graph_add (graph, "storage/bd", volname);
+                if (!xl)
+                        return -1;
+
+                ret = xlator_set_option (xl, "device", "vg");
+                if (ret)
+                        return -1;
+                ret = xlator_set_option (xl, "export", brickinfo->vg);
+                if (ret)
+                        return -1;
+
+                ret = check_and_add_debug_xl (graph, set_dict, volname, "bd");
+                if (ret)
+                        return -1;
+
+        }
+#endif
+
+        xl = volgen_graph_add (graph, "features/changelog", volname);
+        if (!xl)
+                return -1;
+
+        ret = xlator_set_option (xl, "changelog-brick", path);
+        if (ret)
+                return -1;
+
+        snprintf (changelog_basepath, sizeof (changelog_basepath),
+                  "%s/%s", path, ".glusterfs/changelogs");
+        ret = xlator_set_option (xl, "changelog-dir", changelog_basepath);
+        if (ret)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "changelog");
         if (ret)
                 return -1;
 
@@ -1673,23 +1593,20 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                         return -1;
         }
 
-        if (glusterd_is_volume_replicate (volinfo)) {
-                xl = volgen_graph_add (graph, "features/index", volname);
-                if (!xl)
-                        return -1;
-
-                snprintf (index_basepath, sizeof (index_basepath), "%s/%s",
-                          path, ".glusterfs/indices");
-                ret = xlator_set_option (xl, "index-base", index_basepath);
-                if (ret)
-                        return -1;
+        xl = volgen_graph_add (graph, "features/index", volname);
+        if (!xl)
+                return -1;
 
-                ret = check_and_add_debug_xl (graph, set_dict, volname,
-                                              "index");
-                if (ret)
-                        return -1;
+        snprintf (index_basepath, sizeof (index_basepath), "%s/%s",
+                  path, ".glusterfs/indices");
+        ret = xlator_set_option (xl, "index-base", index_basepath);
+        if (ret)
+                return -1;
 
-        }
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "index");
+        if (ret)
+                return -1;
 
         xl = volgen_graph_add (graph, "features/marker", volname);
         if (!xl)
@@ -1708,6 +1625,61 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 return -1;
 
+        xl = volgen_graph_add (graph, "features/quota", volname);
+        if (!xl)
+                return -1;
+        ret = xlator_set_option (xl, "volume-uuid", volname);
+        if (ret)
+                return -1;
+
+        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_QUOTA, &value);
+        if (value) {
+                ret = xlator_set_option (xl, "server-quota", value);
+                if (ret)
+                        return -1;
+        }
+
+         if (dict_get_str_boolean (set_dict, "features.read-only", 0) &&
+            dict_get_str_boolean (set_dict, "features.worm",0)) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "read-only and worm cannot be set together");
+                ret = -1;
+                goto out;
+        }
+
+        /* Check for read-only volume option, and add it to the graph */
+        if (dict_get_str_boolean (set_dict, "features.read-only", 0)) {
+                xl = volgen_graph_add (graph, "features/read-only", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check for worm volume option, and add it to the graph */
+        if (dict_get_str_boolean (set_dict, "features.worm", 0)) {
+                xl = volgen_graph_add (graph, "features/worm", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check for compress volume option, and add it to the graph on server side */
+        ret = dict_get_str_boolean (set_dict, "network.compression", 0);
+        if (ret == -1)
+                goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "features/cdc", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = xlator_set_option (xl, "mode", "server");
+                if (ret)
+                        goto out;
+        }
+
         xl = volgen_graph_add_as (graph, "debug/io-stats", path);
         if (!xl)
                 return -1;
@@ -1719,6 +1691,16 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret)
                 return -1;
 
+        /*In the case of running multiple glusterds on a single machine,
+         * we should ensure that bricks don't listen on all IPs on that
+         * machine and break the IP based separation being brought about.*/
+        if (dict_get (THIS->options, "transport.socket.bind-address")) {
+                ret = xlator_set_option (xl, "transport.socket.bind-address",
+                                         brickinfo->hostname);
+                if (ret)
+                        return -1;
+        }
+
         if (username) {
                 memset (key, 0, sizeof (key));
                 snprintf (key, sizeof (key), "auth.login.%s.allow", path);
@@ -1751,9 +1733,9 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
 /* builds a graph for server role , with option overrides in mod_dict */
 static int
 build_server_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                    dict_t *mod_dict, char *path)
+                    dict_t *mod_dict, glusterd_brickinfo_t *brickinfo)
 {
-        return build_graph_generic (graph, volinfo, mod_dict, path,
+        return build_graph_generic (graph, volinfo, mod_dict, brickinfo,
                                     &server_graph_builder);
 }
 
@@ -1761,12 +1743,42 @@ static int
 perfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
                        void *param)
 {
+        gf_boolean_t enabled = _gf_false;
+        glusterd_volinfo_t *volinfo = NULL;
+
+        GF_ASSERT (param);
+        volinfo = param;
+
+        if (strcmp (vme->option, "!perf") != 0)
+                return 0;
+
+        if (gf_string2boolean (vme->value, &enabled) == -1)
+                return -1;
+        if (!enabled)
+                return 0;
+
+        /* Check op-version before adding the 'open-behind' xlator in the graph
+         */
+        if (!strcmp (vme->key, "performance.open-behind") &&
+            (vme->op_version > volinfo->client_op_version))
+                return 0;
+
+        if (volgen_graph_add (graph, vme->voltype, volinfo->volname))
+                return 0;
+        else
+                return -1;
+}
+
+static int
+nfsperfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
+{
         char *volname = NULL;
         gf_boolean_t enabled = _gf_false;
 
         volname = param;
 
-        if (strcmp (vme->option, "!perf") != 0)
+        if (strcmp (vme->option, "!nfsperf") != 0)
                 return 0;
 
         if (gf_string2boolean (vme->value, &enabled) == -1)
@@ -1916,7 +1928,7 @@ xml_add_volset_element (xmlTextWriterPtr writer, const char *name,
 #endif
 
 static int
-get_key_from_volopt ( struct volopt_map_entry *vme, char **key)
+_get_xlator_opt_key_from_vme ( struct volopt_map_entry *vme, char **key)
 {
         int ret = 0;
 
@@ -1924,25 +1936,32 @@ get_key_from_volopt ( struct volopt_map_entry *vme, char **key)
         GF_ASSERT (key);
 
 
-        if (vme->option) {
-                if  (vme->option[0] == '!') {
-                        *key = vme->option + 1;
-                        if (!*key[0])
-                                ret = -1;
+        if (!strcmp (vme->key, AUTH_ALLOW_MAP_KEY))
+                *key = gf_strdup (AUTH_ALLOW_OPT_KEY);
+        else if (!strcmp (vme->key, AUTH_REJECT_MAP_KEY))
+                *key = gf_strdup (AUTH_REJECT_OPT_KEY);
+        else if (!strcmp (vme->key, NFS_DISABLE_MAP_KEY))
+                *key = gf_strdup (NFS_DISABLE_OPT_KEY);
+        else {
+                if (vme->option) {
+                        if  (vme->option[0] == '!') {
+                                *key = vme->option + 1;
+                                if (!*key[0])
+                                        ret = -1;
+                        } else {
+                                *key = vme->option;
+                        }
                 } else {
-                        *key = vme->option;
-                }
-        } else  {
-                *key = strchr (vme->key, '.');
-                if (*key) {
-                        (*key) ++;
-                        if (!*key[0])
+                        *key = strchr (vme->key, '.');
+                        if (*key) {
+                                (*key) ++;
+                                if (!*key[0])
+                                        ret = -1;
+                        } else {
                                 ret = -1;
-                } else {
-                        ret = -1;
+                        }
                 }
         }
-
         if (ret)
                 gf_log ("glusterd", GF_LOG_ERROR, "Wrong entry found in  "
                         "glusterd_volopt_map entry %s", vme->key);
@@ -1952,23 +1971,32 @@ get_key_from_volopt ( struct volopt_map_entry *vme, char **key)
         return ret;
 }
 
+static void
+_free_xlator_opt_key (char *key)
+{
+        GF_ASSERT (key);
+
+        if (!strcmp (key, AUTH_ALLOW_OPT_KEY) ||
+            !strcmp (key, AUTH_REJECT_OPT_KEY) ||
+            !strcmp (key, NFS_DISABLE_OPT_KEY))
+                GF_FREE (key);
+
+        return;
+}
 
 int
-glusterd_get_volopt_content (gf_boolean_t xml_out)
+glusterd_get_volopt_content (dict_t * ctx, gf_boolean_t xml_out)
 {
-
-        char                    *xlator_type = NULL;
         void                    *dl_handle = NULL;
-        volume_opt_list_t          vol_opt_handle = {{0},};
+        volume_opt_list_t        vol_opt_handle = {{0},};
         char                    *key = NULL;
         struct volopt_map_entry *vme = NULL;
         int                      ret = -1;
         char                    *def_val = NULL;
         char                    *descr = NULL;
-        char                     output_string[8192] = {0, };
+        char                     output_string[51200] = {0, };
         char                    *output = NULL;
-        char                     tmp_str[1024] = {0, };
-        dict_t                  *ctx = NULL;
+        char                     tmp_str[2048] = {0, };
 #if (HAVE_LIB_XML)
         xmlTextWriterPtr         writer = NULL;
         xmlBufferPtr             buf = NULL;
@@ -1980,51 +2008,73 @@ glusterd_get_volopt_content (gf_boolean_t xml_out)
         }
 #endif
 
-        ctx = glusterd_op_get_ctx ();
-
-        if (!ctx) {
-                /*extract the vol-set-help output only in host glusterd*/
-                ret = 0;
-                goto out;
-        }
-
         INIT_LIST_HEAD (&vol_opt_handle.list);
 
         for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
 
-                if ( ( vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC) )
+                if ((vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC))
                         continue;
 
-                if (get_key_from_volopt (vme, &key))
-                                goto out; /*Some error while getin key*/
+                if (vme->description) {
+                        descr = vme->description;
+                        def_val = vme->value;
+                } else {
+                        if (_get_xlator_opt_key_from_vme (vme, &key)) {
+                                gf_log ("glusterd", GF_LOG_DEBUG, "Failed to "
+                                        "get %s key from volume option entry",
+                                        vme->key);
+                                goto out; /*Some error while geting key*/
+                        }
 
-                if (!xlator_type || strcmp (vme->voltype, xlator_type)){
                         ret = xlator_volopt_dynload (vme->voltype,
-                                                        &dl_handle,
-                                                        &vol_opt_handle);
-                        if (ret)
-                                continue;
-                }
+                                                     &dl_handle,
+                                                     &vol_opt_handle);
 
-                ret = xlator_option_info_list (&vol_opt_handle, key,
-                                               &def_val, &descr);
-                if (ret) /*Swallow Error i.e if option not found*/
-                        continue;
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_DEBUG,
+                                        "xlator_volopt_dynload error(%d)", ret);
+                                ret = 0;
+                                goto cont;
+                        }
+
+                        ret = xlator_option_info_list (&vol_opt_handle, key,
+                                                       &def_val, &descr);
+                        if (ret) { /*Swallow Error i.e if option not found*/
+                                gf_log ("glusterd", GF_LOG_DEBUG,
+                                        "Failed to get option for %s key", key);
+                                ret = 0;
+                                goto cont;
+                        }
+                }
 
                 if (xml_out) {
 #if (HAVE_LIB_XML)
                         if (xml_add_volset_element (writer,vme->key,
-                                                        def_val, descr))
-                                goto out;
+                                                    def_val, descr)) {
+                                ret = -1;
+                                goto cont;
+                        }
 #else
                         gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
 #endif
                 } else {
-                        snprintf (tmp_str, 1024, "Option: %s\nDefault "
+                        snprintf (tmp_str, sizeof (tmp_str), "Option: %s\nDefault "
                                         "Value: %s\nDescription: %s\n\n",
                                         vme->key, def_val, descr);
                         strcat (output_string, tmp_str);
                 }
+cont:
+                if (dl_handle) {
+                        dlclose (dl_handle);
+                        dl_handle = NULL;
+                        vol_opt_handle.given_opt = NULL;
+                }
+                if (key) {
+                        _free_xlator_opt_key (key);
+                        key = NULL;
+                }
+                if (ret)
+                        goto out;
         }
 
 #if (HAVE_LIB_XML)
@@ -2051,7 +2101,7 @@ glusterd_get_volopt_content (gf_boolean_t xml_out)
         }
 
         ret = dict_set_dynstr (ctx, "help-str", output);
- out:
+out:
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 
@@ -2069,6 +2119,8 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char                    *str                = NULL;
         glusterd_brickinfo_t    *brick              = NULL;
         xlator_t                *xl                 = NULL;
+        char                    *ssl_str            = NULL;
+        gf_boolean_t             ssl_bool;
 
         volname = volinfo->volname;
 
@@ -2099,7 +2151,7 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         list_for_each_entry (brick, &volinfo->bricks, brick_list) {
                 ret = -1;
                 xl = volgen_graph_add_nolink (graph, "protocol/client",
-                                              "%s-client-%d", volname, i);
+                                              "%s", brick->brick_id);
                 if (!xl)
                         goto out;
                 ret = xlator_set_option (xl, "remote-host", brick->hostname);
@@ -2134,6 +2186,19 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                         }
                 }
 
+                if (dict_get_str(set_dict,"client.ssl",&ssl_str) == 0) {
+                        if (gf_string2boolean(ssl_str,&ssl_bool) == 0) {
+                                if (ssl_bool) {
+                                        ret = xlator_set_option(xl,
+                                                "transport.socket.ssl-enabled",
+                                                "true");
+                                        if (ret) {
+                                                goto out;
+                                        }
+                                }
+                        }
+                }
+
                 i++;
         }
 
@@ -2292,19 +2357,46 @@ out:
 
 static int
 volgen_graph_build_dht_cluster (volgen_graph_t *graph,
-                                glusterd_volinfo_t *volinfo, size_t child_count)
+                                glusterd_volinfo_t *volinfo, size_t child_count,
+                                gf_boolean_t is_quotad)
 {
         int32_t                 clusters                 = 0;
         int                     ret                      = -1;
         char                    *decommissioned_children = NULL;
         xlator_t                *dht                     = NULL;
+        char                    *voltype                 = "cluster/distribute";
+        char                    *name_fmt                = NULL;
+
+        /* NUFA and Switch section */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.nufa", 0) &&
+            dict_get_str_boolean (volinfo->dict, "cluster.switch", 0)) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "nufa and switch cannot be set together");
+                ret = -1;
+                goto out;
+        }
+
+        /* Check for NUFA volume option, and change the voltype */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.nufa", 0))
+                voltype = "cluster/nufa";
+
+        /* Check for switch volume option, and change the voltype */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.switch", 0))
+                voltype = "cluster/switch";
+
+        if (is_quotad)
+                name_fmt = "%s";
+        else
+                name_fmt = "%s-dht";
 
-        GF_ASSERT (child_count > 1);
         clusters = volgen_graph_build_clusters (graph,  volinfo,
-                                                "cluster/distribute", "%s-dht",
-                                                child_count, child_count);
+                                                voltype,
+                                                name_fmt,
+                                                child_count,
+                                                child_count);
         if (clusters < 0)
                 goto out;
+
         dht = first_of (graph);
         ret = _graph_get_decommissioned_children (dht, volinfo,
                                                   &decommissioned_children);
@@ -2318,14 +2410,14 @@ volgen_graph_build_dht_cluster (volgen_graph_t *graph,
         }
         ret = 0;
 out:
-        if (decommissioned_children)
-                GF_FREE (decommissioned_children);
+        GF_FREE (decommissioned_children);
         return ret;
 }
 
 static int
 volume_volgen_graph_build_clusters (volgen_graph_t *graph,
-                                    glusterd_volinfo_t *volinfo)
+                                    glusterd_volinfo_t *volinfo,
+                                    gf_boolean_t is_quotad)
 {
         char                    *replicate_args[]   = {"cluster/replicate",
                                                        "%s-replicate-%d"};
@@ -2392,39 +2484,115 @@ volume_volgen_graph_build_clusters (volgen_graph_t *graph,
 
 build_distribute:
         dist_count = volinfo->brick_count / volinfo->dist_leaf_count;
-        if (dist_count > 1) {
-                ret = volgen_graph_build_dht_cluster (graph, volinfo,
-                                                      dist_count);
-                if (ret)
-                        goto out;
+        if (!dist_count) {
+                ret = -1;
+                goto out;
         }
+
+        ret = volgen_graph_build_dht_cluster (graph, volinfo,
+                                              dist_count, is_quotad);
+        if (ret)
+                goto out;
+
         ret = 0;
 out:
         return ret;
 }
 
+static int client_graph_set_perf_options(volgen_graph_t *graph,
+					 glusterd_volinfo_t *volinfo,
+					 dict_t *set_dict)
+{
+	data_t *tmp_data = NULL;
+	char *volname = NULL;
+
+	/*
+	 * Logic to make sure NFS doesn't have performance translators by
+	 * default for a volume
+	 */
+	volname = volinfo->volname;
+	tmp_data = dict_get (set_dict, "nfs-volume-file");
+	if (!tmp_data)
+		return volgen_graph_set_options_generic(graph, set_dict,
+							volname,
+							&perfxl_option_handler);
+	else
+		return volgen_graph_set_options_generic(graph, set_dict,
+							volname,
+							&nfsperfxl_option_handler);
+}
+
 static int
 client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                       dict_t *set_dict, void *param)
 {
-        int                      ret                = 0;
-        xlator_t                *xl                 = NULL;
-        char                    *volname            = NULL;
+        int              ret     = 0;
+        xlator_t        *xl      = NULL;
+        char            *volname = NULL;
+        glusterd_conf_t *conf    = THIS->private;
+        char            *tmp     = NULL;
+        gf_boolean_t     var     = _gf_false;
+        gf_boolean_t     ob      = _gf_false;
+
+        GF_ASSERT (conf);
 
         volname = volinfo->volname;
         ret = volgen_graph_build_clients (graph, volinfo, set_dict, param);
         if (ret)
                 goto out;
 
-        ret = volume_volgen_graph_build_clusters (graph, volinfo);
-        if (ret)
+        ret = volume_volgen_graph_build_clusters (graph, volinfo, _gf_false);
+        if (ret == -1)
+                goto out;
+
+        /* Check for compress volume option, and add it to the graph on client side */
+        ret = dict_get_str_boolean (set_dict, "network.compression", 0);
+        if (ret == -1)
+                goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "features/cdc", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = xlator_set_option (xl, "mode", "client");
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_volinfo_get_boolean (volinfo, "features.encryption");
+        if (ret == -1)
                 goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "encryption/crypt", volname);
+
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (conf->op_version == GD_OP_VERSION_MIN) {
+                ret = glusterd_volinfo_get_boolean (volinfo,
+                                                    VKEY_FEATURES_QUOTA);
+                if (ret == -1)
+                        goto out;
+                if (ret) {
+                        xl = volgen_graph_add (graph, "features/quota",
+                                              volname);
+                        if (!xl) {
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
 
-        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
+        ret = glusterd_volinfo_get_boolean (volinfo, "features.file-snapshot");
         if (ret == -1)
                 goto out;
         if (ret) {
-                xl = volgen_graph_add (graph, "features/quota", volname);
+                xl = volgen_graph_add (graph, "features/qemu-block", volname);
 
                 if (!xl) {
                         ret = -1;
@@ -2432,11 +2600,102 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 }
         }
 
-        ret = volgen_graph_set_options_generic (graph, set_dict, volname,
-                                                &perfxl_option_handler);
+        /* Do not allow changing read-after-open option if root-squash is
+           enabled.
+        */
+        ret = dict_get_str (set_dict, "performance.read-after-open", &tmp);
+        if (!ret) {
+                ret = dict_get_str (volinfo->dict, "server.root-squash", &tmp);
+                if (!ret) {
+                        ob = _gf_false;
+                        ret = gf_string2boolean (tmp, &ob);
+                        if (!ret && ob) {
+                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        "root-squash is enabled. Please turn it"
+                                        " off to change read-after-open "
+                                        "option");
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        /* open behind causes problems when root-squash is enabled
+           (by allowing reads to happen even though the squashed user
+           does not have permissions to do so) as it fakes open to be
+           successful and later sends reads on anonymous fds. So when
+           root-squash is enabled, open-behind's option to read after
+           open is done is also enabled.
+        */
+        ret = dict_get_str (set_dict, "server.root-squash", &tmp);
+        if (!ret) {
+                ret = gf_string2boolean (tmp, &var);
+                if (ret)
+                        goto out;
+
+                if (var) {
+                        ret = dict_get_str (volinfo->dict,
+                                            "performance.read-after-open",
+                                            &tmp);
+                        if (!ret) {
+                                ret = gf_string2boolean (tmp, &ob);
+                                /* go ahead with turning read-after-open on
+                                   even if string2boolean conversion fails,
+                                   OR if read-after-open option is turned off
+                                */
+                                if (ret || !ob)
+                                        ret = dict_set_str (set_dict,
+                                                 "performance.read-after-open",
+                                                            "yes");
+                        } else {
+                                ret = dict_set_str (set_dict,
+                                                    "performance.read-after-open",
+                                                    "yes");
+                        }
+                } else {
+                        /* When root-squash has to be turned off, open-behind's
+                           read-after-open option should be reset to what was
+                           there before root-squash was turned on. If the option
+                           cannot be found in volinfo's dict, it means that
+                           option was not set before turning on root-squash.
+                        */
+                        ob = _gf_false;
+                        ret = dict_get_str (volinfo->dict,
+                                            "performance.read-after-open",
+                                            &tmp);
+                        if (!ret) {
+                                ret = gf_string2boolean (tmp, &ob);
+
+                                if (!ret && ob) {
+                                        ret = dict_set_str (set_dict,
+                                                 "performance.read-after-open",
+                                                            "yes");
+                                }
+                        }
+                        /* consider operation is failure only if read-after-open
+                           option is enabled and could not set into set_dict
+                        */
+                        if (!ob)
+                                ret = 0;
+                }
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_WARNING, "setting "
+                                "open behind option as part of root "
+                                "squash failed");
+                        goto out;
+                }
+        }
+
+        ret = client_graph_set_perf_options(graph, volinfo, set_dict);
         if (ret)
                 goto out;
 
+        /* add debug translators depending on the options */
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "client");
+        if (ret)
+                return -1;
+
         ret = -1;
         xl = volgen_graph_add_as (graph, "debug/io-stats", volname);
         if (!xl)
@@ -2445,9 +2704,15 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         ret = volgen_graph_set_options_generic (graph, set_dict, "client",
                                                 &loglevel_option_handler);
 
-        if (!ret)
-                ret = volgen_graph_set_options_generic (graph, set_dict, "client",
-                                                        &sys_loglevel_option_handler);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING, "changing client log level"
+                        " failed");
+
+        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
+                                                &sys_loglevel_option_handler);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING, "changing client syslog "
+                        "level failed");
 out:
         return ret;
 }
@@ -2462,20 +2727,35 @@ build_client_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                                     &client_graph_builder);
 }
 
+char *gd_shd_options[] = {
+        "!self-heal-daemon",
+        "!heal-timeout",
+        NULL
+};
+
+char*
+gd_get_matching_option (char **options, char *option)
+{
+        while (*options && strcmp (*options, option))
+                options++;
+        return *options;
+}
+
 static int
 shd_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
                     void *param)
 {
         int                     ret = 0;
         struct volopt_map_entry new_vme = {0};
-        int                     shd = 0;
+        char                    *shd_option = NULL;
 
-        shd = !strcmp (vme->option, "!self-heal-daemon");
-        if ((vme->option[0] == '!') && !shd)
+        shd_option = gd_get_matching_option (gd_shd_options, vme->option);
+        if ((vme->option[0] == '!') && !shd_option)
                 goto out;
         new_vme = *vme;
-        if (shd)
-                new_vme.option = "self-heal-daemon";
+        if (shd_option) {
+                new_vme.option = shd_option + 1;//option with out '!'
+        }
 
         ret = no_filter_option_handler (graph, &new_vme, param);
 out:
@@ -2499,7 +2779,7 @@ nfs_option_handler (volgen_graph_t *graph,
 
                 ret = xlator_set_option (xl, vme->key, vme->value);
         }*/
-        if ( !volinfo || !volinfo->volname)
+        if (!volinfo || (volinfo->volname[0] == '\0'))
                 return 0;
 
         if (! strcmp (vme->option, "!rpc-auth.addr.*.allow")) {
@@ -2529,7 +2809,7 @@ nfs_option_handler (volgen_graph_t *graph,
         }
 
         if (! strcmp (vme->option, "!rpc-auth.auth-unix.*")) {
-                ret = gf_asprintf (&aa, "rpc-auth.auth.unix.%s",
+                ret = gf_asprintf (&aa, "rpc-auth.auth-unix.%s",
                                         volinfo->volname);
 
                 if (ret != -1) {
@@ -2540,8 +2820,8 @@ nfs_option_handler (volgen_graph_t *graph,
                 if (ret)
                         return -1;
         }
-        if (! strcmp (vme->option, "!rpc-auth.auth.null.*")) {
-                ret = gf_asprintf (&aa, "rpc-auth.auth.null.%s",
+        if (! strcmp (vme->option, "!rpc-auth.auth-null.*")) {
+                ret = gf_asprintf (&aa, "rpc-auth.auth-null.%s",
                                         volinfo->volname);
 
                 if (ret != -1) {
@@ -2597,6 +2877,10 @@ nfs_option_handler (volgen_graph_t *graph,
                                         volinfo->volname);
 
                 if (ret != -1) {
+                        ret = gf_canonicalize_path (vme->value);
+                        if (ret)
+                                return -1;
+
                         ret = xlator_set_option (xl, aa, vme->value);
                         GF_FREE (aa);
                 }
@@ -2688,6 +2972,7 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
         xlator_t           *iostxl        = NULL;
         int                rclusters      = 0;
         int                replica_count  = 0;
+        gf_boolean_t       graph_check    = _gf_false;
 
         this = THIS;
         priv = this->private;
@@ -2698,6 +2983,7 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 goto out;
         }
 
+        graph_check = dict_get_str_boolean (mod_dict, "graph-check", 0);
         iostxl = volgen_graph_add_as (graph, "debug/io-stats", "glustershd");
         if (!iostxl) {
                 ret = -1;
@@ -2705,7 +2991,8 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
         }
 
         list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (voliter->status != GLUSTERD_STATUS_STARTED)
+                if (!graph_check &&
+                   (voliter->status != GLUSTERD_STATUS_STARTED))
                         continue;
 
                 if (!glusterd_is_volume_replicate (voliter))
@@ -2757,6 +3044,21 @@ build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 if (ret)
                         goto out;
 
+                ret = volgen_graph_set_options_generic (graph, set_dict,
+                                                        "client",
+                                                 &loglevel_option_handler);
+
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING, "changing loglevel "
+                                "of self-heal daemon failed");
+
+                ret = volgen_graph_set_options_generic (graph, set_dict,
+                                                        "client",
+                                                 &sys_loglevel_option_handler);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING, "changing syslog "
+                                "level of self-heal daemon failed");
+
                 ret = dict_reset (set_dict);
                 if (ret)
                         goto out;
@@ -2782,6 +3084,8 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
         char               *skey          = NULL;
         int                 ret           = 0;
         char               nfs_xprt[16]   = {0,};
+        char               *volname       = NULL;
+        data_t             *data          = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -2803,6 +3107,14 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
         if (ret)
                 goto out;
 
+        ret = xlator_set_option (nfsxl, "nfs.nlm", "on");
+        if (ret)
+                goto out;
+
+        ret = xlator_set_option (nfsxl, "nfs.drc", "on");
+        if (ret)
+                goto out;
+
         list_for_each_entry (voliter, &priv->volumes, vol_list) {
                 if (voliter->status != GLUSTERD_STATUS_STARTED)
                         continue;
@@ -2840,7 +3152,7 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 else
                         get_transport_type (voliter, voliter->dict, nfs_xprt, _gf_true);
 
-                ret = dict_set_str (set_dict, VKEY_PERF_STAT_PREFETCH, "off");
+                ret = dict_set_str (set_dict, "performance.stat-prefetch", "off");
                 if (ret)
                         goto out;
 
@@ -2859,6 +3171,16 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
                 if (ret)
                         goto out;
 
+                ret = dict_set_str (set_dict, "nfs-volume-file", "yes");
+                if (ret)
+                        goto out;
+
+                if (mod_dict && (data = dict_get (mod_dict, "volume-name"))) {
+                        volname = data->data;
+                        if (strcmp (volname, voliter->volname) == 0)
+                                dict_copy (mod_dict, set_dict);
+                }
+
                 ret = build_client_graph (&cgraph, voliter, set_dict);
                 if (ret)
                         goto out;
@@ -2905,9 +3227,6 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
         return ret;
 }
 
-
-
-
 /****************************
  *
  * Volume generation interface
@@ -2941,16 +3260,21 @@ glusterd_is_valid_volfpath (char *volname, char *brick)
         glusterd_brickinfo_t    *brickinfo = NULL;
         glusterd_volinfo_t      *volinfo = NULL;
         int32_t                 ret = 0;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
-        ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
+        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "brick path validation failed");
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create brickinfo"
+                        " for brick %s", brick );
                 ret = 0;
                 goto out;
         }
         ret = glusterd_volinfo_new (&volinfo);
         if (ret) {
-                gf_log ("", GF_LOG_WARNING, "brick path validation failed");
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create volinfo");
                 ret = 0;
                 goto out;
         }
@@ -2963,7 +3287,7 @@ out:
         if (brickinfo)
                 glusterd_brickinfo_delete (brickinfo);
         if (volinfo)
-                glusterd_volinfo_delete (volinfo);
+                glusterd_volinfo_unref (volinfo);
         return ret;
 }
 
@@ -2980,7 +3304,7 @@ glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
 
         get_brick_filepath (filename, volinfo, brickinfo);
 
-        ret = build_server_graph (&graph, volinfo, NULL, brickinfo->path);
+        ret = build_server_graph (&graph, volinfo, NULL, brickinfo);
         if (!ret)
                 ret = volgen_write_volfile (&graph, filename);
 
@@ -2989,7 +3313,100 @@ glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
         return ret;
 }
 
+static int
+build_quotad_graph (volgen_graph_t *graph, dict_t *mod_dict)
+{
+        volgen_graph_t     cgraph         = {0};
+        glusterd_volinfo_t *voliter       = NULL;
+        xlator_t           *this          = NULL;
+        glusterd_conf_t    *priv          = NULL;
+        dict_t             *set_dict      = NULL;
+        int                ret            = 0;
+        xlator_t           *quotad_xl     = NULL;
+	char		   *skey	  = NULL;
+
+        this = THIS;
+        priv = this->private;
+
+        set_dict = dict_new ();
+        if (!set_dict) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        quotad_xl = volgen_graph_add_as (graph, "features/quotad", "quotad");
+        if (!quotad_xl) {
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (voliter->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+
+                if (1 != glusterd_is_volume_quota_enabled (voliter))
+                        continue;
+
+                ret = dict_set_uint32 (set_dict, "trusted-client",
+                                       GF_CLIENT_TRUSTED);
+                if (ret)
+                        goto out;
+
+                dict_copy (voliter->dict, set_dict);
+                if (mod_dict)
+                        dict_copy (mod_dict, set_dict);
+
+                ret = gf_asprintf(&skey, "%s.volume-id", voliter->volname);
+                if (ret == -1) {
+                        gf_log("", GF_LOG_ERROR, "Out of memory");
+                        goto out;
+                }
+                ret = xlator_set_option(quotad_xl, skey, voliter->volname);
+                GF_FREE(skey);
+                if (ret)
+                        goto out;
+
+                memset (&cgraph, 0, sizeof (cgraph));
+                ret = volgen_graph_build_clients (&cgraph, voliter, set_dict,
+                                                  NULL);
+                if (ret)
+                        goto out;
+
+                ret = volume_volgen_graph_build_clusters (&cgraph, voliter,
+                                                          _gf_true);
+                if (ret) {
+                        ret = -1;
+                        goto out;
+                }
+
+                if (mod_dict) {
+                        dict_copy (mod_dict, set_dict);
+                        ret = volgen_graph_set_options_generic (&cgraph, set_dict,
+                                                        voliter,
+                                                        basic_option_handler);
+                } else {
+                        ret = volgen_graph_set_options_generic (&cgraph,
+                                                                voliter->dict,
+                                                                voliter,
+                                                                basic_option_handler);
+                }
+                if (ret)
+                        goto out;
+
+                ret = volgen_graph_merge_sub (graph, &cgraph, 1);
+                if (ret)
+                        goto out;
+
+                ret = dict_reset (set_dict);
+                if (ret)
+                        goto out;
+        }
 
+out:
+        if (set_dict)
+                dict_unref (set_dict);
+        return ret;
+}
 
 static void
 get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo)
@@ -3091,7 +3508,7 @@ enumerate_transport_reqs (gf_transport_type type, char **types)
         }
 }
 
-static int
+int
 generate_client_volfiles (glusterd_volinfo_t *volinfo,
                           glusterd_client_type_t client_type)
 {
@@ -3238,6 +3655,14 @@ glusterd_create_shd_volfile ()
         if (ret)
                 goto out;
 
+        ret = dict_set_str (mod_dict, "cluster.metadata-self-heal", "on");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (mod_dict, "cluster.entry-self-heal", "on");
+        if (ret)
+                goto out;
+
         glusterd_get_nodesvc_volfile ("glustershd", conf->workdir,
                                       filepath, sizeof (filepath));
         ret = glusterd_create_global_volfile (build_shd_graph, filepath,
@@ -3249,6 +3674,118 @@ out:
 }
 
 int
+glusterd_check_nfs_volfile_identical (gf_boolean_t *identical)
+{
+        char            nfsvol[PATH_MAX]        = {0,};
+        char            tmpnfsvol[PATH_MAX]     = {0,};
+        glusterd_conf_t *conf                   = NULL;
+        xlator_t        *this                   = NULL;
+        int             ret                     = -1;
+        int             need_unlink             = 0;
+        int             tmp_fd                  = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (identical);
+        conf = this->private;
+
+        glusterd_get_nodesvc_volfile ("nfs", conf->workdir,
+                                      nfsvol, sizeof (nfsvol));
+
+        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
+
+        tmp_fd = mkstemp (tmpnfsvol);
+        if (tmp_fd < 0) {
+                gf_log ("", GF_LOG_WARNING, "Unable to create temp file %s: "
+                                "(%s)", tmpnfsvol, strerror (errno));
+                goto out;
+        }
+
+        need_unlink = 1;
+
+        ret = glusterd_create_global_volfile (build_nfs_graph,
+                                              tmpnfsvol, NULL);
+        if (ret)
+                goto out;
+
+        ret = glusterd_check_files_identical (nfsvol, tmpnfsvol,
+                                              identical);
+        if (ret)
+                goto out;
+
+out:
+        if (need_unlink)
+                unlink (tmpnfsvol);
+
+        if (tmp_fd >= 0)
+                close (tmp_fd);
+
+        return ret;
+}
+
+int
+glusterd_check_nfs_topology_identical (gf_boolean_t *identical)
+{
+        char            nfsvol[PATH_MAX]        = {0,};
+        char            tmpnfsvol[PATH_MAX]     = {0,};
+        glusterd_conf_t *conf                   = NULL;
+        xlator_t        *this                   = THIS;
+        int             ret                     = -1;
+        int             tmpclean                = 0;
+        int             tmpfd                   = -1;
+
+        if ((!identical) || (!this) || (!this->private))
+                goto out;
+
+        conf = (glusterd_conf_t *) this->private;
+
+        /* Fetch the original NFS volfile */
+        glusterd_get_nodesvc_volfile ("nfs", conf->workdir,
+                                      nfsvol, sizeof (nfsvol));
+
+        /* Create the temporary NFS volfile */
+        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
+        tmpfd = mkstemp (tmpnfsvol);
+        if (tmpfd < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                                    "Unable to create temp file %s: (%s)",
+                                    tmpnfsvol, strerror (errno));
+                goto out;
+        }
+
+        tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+        ret = glusterd_create_global_volfile (build_nfs_graph,
+                                              tmpnfsvol, NULL);
+        if (ret)
+                goto out;
+
+        /* Compare the topology of volfiles */
+        ret = glusterd_check_topology_identical (nfsvol, tmpnfsvol,
+                                                 identical);
+out:
+        if (tmpfd >= 0)
+                close (tmpfd);
+        if (tmpclean)
+                unlink (tmpnfsvol);
+        return ret;
+}
+
+int
+glusterd_create_quotad_volfile ()
+{
+        char             filepath[PATH_MAX] = {0,};
+        glusterd_conf_t *conf               = THIS->private;
+
+        glusterd_get_nodesvc_volfile ("quotad", conf->workdir,
+                                            filepath, sizeof (filepath));
+        return glusterd_create_global_volfile (build_quotad_graph,
+                                               filepath, NULL);
+}
+
+
+int
 glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
                          glusterd_brickinfo_t *brickinfo)
 {
@@ -3267,6 +3804,35 @@ glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
 }
 
 int
+validate_shdopts (glusterd_volinfo_t *volinfo,
+                  dict_t *val_dict,
+                  char **op_errstr)
+{
+        volgen_graph_t graph = {0,};
+        int     ret = -1;
+
+        graph.errstr = op_errstr;
+
+        if (!glusterd_is_volume_replicate (volinfo)) {
+                ret = 0;
+                goto out;
+        }
+        ret = dict_set_str (val_dict, "graph-check", "on");
+        if (ret)
+                goto out;
+        ret = build_shd_graph (&graph, val_dict);
+        if (!ret)
+                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+
+        volgen_graph_free (&graph);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+out:
+        dict_del (val_dict, "graph-check");
+        return ret;
+}
+
+int
 validate_nfsopts (glusterd_volinfo_t *volinfo,
                     dict_t *val_dict,
                     char **op_errstr)
@@ -3276,6 +3842,9 @@ validate_nfsopts (glusterd_volinfo_t *volinfo,
         char    transport_type[16] = {0,};
         char    *tt                = NULL;
         char    err_str[4096]      = {0,};
+        xlator_t *this             = THIS;
+
+        GF_ASSERT (this);
 
         graph.errstr = op_errstr;
 
@@ -3286,7 +3855,7 @@ validate_nfsopts (glusterd_volinfo_t *volinfo,
                         snprintf (err_str, sizeof (err_str), "Changing nfs "
                                   "transport type is allowed only for volumes "
                                   "of transport type tcp,rdma");
-                        gf_log ("", GF_LOG_ERROR, "%s", err_str);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                         *op_errstr = gf_strdup (err_str);
                         ret = -1;
                         goto out;
@@ -3300,6 +3869,12 @@ validate_nfsopts (glusterd_volinfo_t *volinfo,
                 }
         }
 
+        ret = dict_set_str (val_dict, "volume-name", volinfo->volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volume name");
+                goto out;
+        }
+
         ret = build_nfs_graph (&graph, val_dict);
         if (!ret)
                 ret = graph_reconf_validateopt (&graph.graph, op_errstr);
@@ -3307,10 +3882,13 @@ validate_nfsopts (glusterd_volinfo_t *volinfo,
         volgen_graph_free (&graph);
 
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        if (dict_get (val_dict, "volume-name"))
+                dict_del (val_dict, "volume-name");
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
+
 int
 validate_clientopts (glusterd_volinfo_t *volinfo,
                      dict_t *val_dict,
@@ -3335,7 +3913,7 @@ validate_clientopts (glusterd_volinfo_t *volinfo,
 
 int
 validate_brickopts (glusterd_volinfo_t *volinfo,
-                    char *brickinfo_path,
+                    glusterd_brickinfo_t *brickinfo,
                     dict_t *val_dict,
                     char **op_errstr)
 {
@@ -3346,7 +3924,7 @@ validate_brickopts (glusterd_volinfo_t *volinfo,
 
         graph.errstr = op_errstr;
 
-        ret = build_server_graph (&graph, volinfo, val_dict, brickinfo_path);
+        ret = build_server_graph (&graph, volinfo, val_dict, brickinfo);
         if (!ret)
                 ret = graph_reconf_validateopt (&graph.graph, op_errstr);
 
@@ -3368,7 +3946,7 @@ glusterd_validate_brickreconf (glusterd_volinfo_t *volinfo,
                 gf_log ("", GF_LOG_DEBUG,
                         "Validating %s", brickinfo->hostname);
 
-                ret = validate_brickopts (volinfo, brickinfo->path, val_dict,
+                ret = validate_brickopts (volinfo, brickinfo, val_dict,
                                           op_errstr);
                 if (ret)
                         goto out;
@@ -3380,16 +3958,18 @@ out:
         return ret;
 }
 
-static void
+static int
 _check_globalopt (dict_t *this, char *key, data_t *value, void *ret_val)
 {
         int *ret = NULL;
 
         ret = ret_val;
         if (*ret)
-                return;
+                return 0;
         if (!glusterd_check_globaloption (key))
                 *ret = 1;
+
+        return 0;
 }
 
 int
@@ -3419,22 +3999,34 @@ glusterd_validate_globalopts (glusterd_volinfo_t *volinfo,
         }
 
         ret = validate_nfsopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate nfs");
+                goto out;
+        }
+
+        ret = validate_shdopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate self-heald");
+                goto out;
+        }
 
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
-static void
+static int
 _check_localopt (dict_t *this, char *key, data_t *value, void *ret_val)
 {
         int *ret = NULL;
 
         ret = ret_val;
         if (*ret)
-                return;
+                return 0;
         if (!glusterd_check_localoption (key))
                 *ret = 1;
+
+        return 0;
 }
 
 int
@@ -3464,9 +4056,129 @@ glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict,
         }
 
         ret = validate_nfsopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate nfs");
+                goto out;
+        }
+
+
+        ret = validate_shdopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate self-heald");
+                goto out;
+        }
 
 
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+
+static struct volopt_map_entry *
+_gd_get_vmep (char *key) {
+        char *completion = NULL;
+        struct volopt_map_entry *vmep = NULL;
+        int ret = 0;
+
+        COMPLETE_OPTION ((char *)key, completion, ret);
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0)
+                        return vmep;
+        }
+
+        return NULL;
+}
+
+uint32_t
+glusterd_get_op_version_for_key (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep)
+                return vmep->op_version;
+
+        return 0;
+}
+
+gf_boolean_t
+gd_is_client_option (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep && (vmep->flags & OPT_FLAG_CLIENT_OPT))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+gf_boolean_t
+gd_is_xlator_option (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep && (vmep->flags & OPT_FLAG_XLATOR_OPT))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+volume_option_type_t
+_gd_get_option_type (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+        void                    *dl_handle = NULL;
+        volume_opt_list_t       vol_opt_list = {{0},};
+        int                     ret = -1;
+        volume_option_t         *opt = NULL;
+        char                    *xlopt_key = NULL;
+        volume_option_type_t    opt_type = GF_OPTION_TYPE_MAX;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+
+        if (vmep) {
+                INIT_LIST_HEAD (&vol_opt_list.list);
+                ret = xlator_volopt_dynload (vmep->voltype, &dl_handle,
+                                             &vol_opt_list);
+                if (ret)
+                        goto out;
+
+                if (_get_xlator_opt_key_from_vme (vmep, &xlopt_key))
+                        goto out;
+
+                opt = xlator_volume_option_get_list (&vol_opt_list, xlopt_key);
+                _free_xlator_opt_key (xlopt_key);
+
+                if (opt)
+                        opt_type = opt->type;
+        }
+
+out:
+        if (dl_handle) {
+                dlclose (dl_handle);
+                dl_handle = NULL;
+        }
+
+        return opt_type;
+}
+
+gf_boolean_t
+gd_is_boolean_option (char *key)
+{
+        GF_ASSERT (key);
+
+        if (GF_OPTION_TYPE_BOOL == _gd_get_option_type (key))
+                return _gf_true;
+
+        return _gf_false;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index feb1796c9..1683f9050 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_VOLGEN_H_
 #define _GLUSTERD_VOLGEN_H_
 
@@ -32,9 +22,19 @@
 #define VKEY_DIAG_CNT_FOP_HITS    "diagnostics.count-fop-hits"
 #define VKEY_DIAG_LAT_MEASUREMENT "diagnostics.latency-measurement"
 #define VKEY_FEATURES_LIMIT_USAGE "features.limit-usage"
+#define VKEY_FEATURES_SOFT_LIMIT  "features.soft-limit"
 #define VKEY_MARKER_XTIME         GEOREP".indexing"
+#define VKEY_MARKER_XTIME_FORCE   GEOREP".ignore-pid-check"
+#define VKEY_CHANGELOG            "changelog.changelog"
 #define VKEY_FEATURES_QUOTA       "features.quota"
-#define VKEY_PERF_STAT_PREFETCH   "performance.stat-prefetch"
+
+#define AUTH_ALLOW_MAP_KEY "auth.allow"
+#define AUTH_REJECT_MAP_KEY "auth.reject"
+#define NFS_DISABLE_MAP_KEY "nfs.disable"
+#define AUTH_ALLOW_OPT_KEY "auth.addr.*.allow"
+#define AUTH_REJECT_OPT_KEY "auth.addr.*.reject"
+#define NFS_DISABLE_OPT_KEY "nfs.*.disable"
+
 
 typedef enum {
         GF_CLIENT_TRUSTED,
@@ -63,9 +63,54 @@ typedef enum {
 
 typedef enum gd_volopt_flags_ {
         OPT_FLAG_NONE,
-        OPT_FLAG_FORCE = 1,
+        OPT_FLAG_FORCE = 0x01,      // option needs force to be reset
+        OPT_FLAG_XLATOR_OPT = 0x02, // option enables/disables xlators
+        OPT_FLAG_CLIENT_OPT = 0x04, // option affects clients
 } gd_volopt_flags_t;
 
+typedef enum {
+        GF_XLATOR_POSIX = 0,
+        GF_XLATOR_ACL,
+        GF_XLATOR_LOCKS,
+        GF_XLATOR_IOT,
+        GF_XLATOR_INDEX,
+        GF_XLATOR_MARKER,
+        GF_XLATOR_IO_STATS,
+        GF_XLATOR_BD,
+        GF_XLATOR_NONE,
+} glusterd_server_xlator_t;
+
+/* As of now debug xlators can be loaded only below fuse in the client
+ * graph via cli. More xlators can be added below when the cli option
+ * for adding debug xlators anywhere in the client graph has to be made
+ * available.
+ */
+typedef enum {
+        GF_CLNT_XLATOR_FUSE = 0,
+        GF_CLNT_XLATOR_NONE,
+} glusterd_client_xlator_t;
+
+typedef enum  { DOC, NO_DOC, GLOBAL_DOC, GLOBAL_NO_DOC } option_type_t;
+
+typedef int (*vme_option_validation) (dict_t *dict, char *key, char *value,
+                                      char **op_errstr);
+
+struct volopt_map_entry {
+        char *key;
+        char *voltype;
+        char *option;
+        char *value;
+        option_type_t type;
+        uint32_t flags;
+        uint32_t op_version;
+        char *description;
+        vme_option_validation validate_fn;
+        /* If client_option is true, the option affects clients.
+         * this is used to calculate client-op-version of volumes
+         */
+        //gf_boolean_t client_option;
+};
+
 int glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
                                  glusterd_brickinfo_t *brickinfo);
 
@@ -77,6 +122,7 @@ void glusterd_get_shd_filepath (char *filename);
 
 int glusterd_create_nfs_volfile ();
 int glusterd_create_shd_volfile ();
+int glusterd_create_quotad_volfile ();
 
 int glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
                              glusterd_brickinfo_t *brickinfo);
@@ -93,7 +139,26 @@ glusterd_check_voloption_flags (char *key, int32_t flags);
 gf_boolean_t
 glusterd_is_valid_volfpath (char *volname, char *brick);
 int generate_brick_volfiles (glusterd_volinfo_t *volinfo);
-int glusterd_get_volopt_content (gf_boolean_t xml_out);
+int generate_client_volfiles (glusterd_volinfo_t *volinfo,
+                              glusterd_client_type_t client_type);
+int glusterd_get_volopt_content (dict_t *dict, gf_boolean_t xml_out);
 char*
 glusterd_get_trans_type_rb (gf_transport_type ttype);
+int
+glusterd_check_nfs_volfile_identical (gf_boolean_t *identical);
+int
+glusterd_check_nfs_topology_identical (gf_boolean_t *identical);
+
+uint32_t
+glusterd_get_op_version_for_key (char *key);
+
+gf_boolean_t
+gd_is_client_option (char *key);
+
+gf_boolean_t
+gd_is_xlator_option (char *key);
+
+gf_boolean_t
+gd_is_boolean_option (char *key);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 2d06ad834..135faa40a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1,27 +1,21 @@
 /*
-  Copyright (c) 2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
 #include "common-utils.h"
 #include "syscall.h"
 #include "cli1-xdr.h"
@@ -36,20 +30,18 @@
 #define glusterd_op_start_volume_args_get(dict, volname, flags) \
         glusterd_op_stop_volume_args_get (dict, volname, flags)
 
+extern int
+_get_slave_status (dict_t *this, char *key, data_t *value, void *data);
+
 int
-glusterd_handle_create_volume (rpcsvc_request_t *req)
+__glusterd_handle_create_volume (rpcsvc_request_t *req)
 {
         int32_t                 ret         = -1;
         gf_cli_req              cli_req     = {{0,}};
         dict_t                 *dict        = NULL;
-        glusterd_brickinfo_t   *brickinfo   = NULL;
-        char                   *brick       = NULL;
         char                   *bricks      = NULL;
         char                   *volname     = NULL;
         int                    brick_count  = 0;
-        char                   *tmpptr      = NULL;
-        int                    i            = 0;
-        char                   *brick_list  = NULL;
         void                   *cli_rsp     = NULL;
         char                    err_str[2048] = {0,};
         gf_cli_rsp              rsp         = {0,};
@@ -58,29 +50,26 @@ glusterd_handle_create_volume (rpcsvc_request_t *req)
         char                   *trans_type  = NULL;
         uuid_t                  volume_id   = {0,};
         uuid_t                  tmp_uuid    = {0};
-        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
-        glusterd_volinfo_t      tmpvolinfo  = {{0},};
         int32_t                 type        = 0;
         char                   *username    = NULL;
         char                   *password    = NULL;
 
         GF_ASSERT (req);
 
-        INIT_LIST_HEAD (&tmpvolinfo.bricks);
-
         this = THIS;
         GF_ASSERT(this);
 
         ret = -1;
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
-                snprintf (err_str, sizeof (err_str), "Garbage args received");
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received create volume req");
+        gf_log (this->name, GF_LOG_DEBUG, "Received create volume req");
 
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
@@ -90,11 +79,11 @@ glusterd_handle_create_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
                         snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the buffer");
+                                  "the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -104,47 +93,52 @@ glusterd_handle_create_volume (rpcsvc_request_t *req)
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
                 snprintf (err_str, sizeof (err_str), "Unable to get volume "
                           "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
-        gf_cmd_log ("Volume create", "on volname: %s attempted", volname);
 
         if ((ret = glusterd_check_volume_exists (volname))) {
-                snprintf(err_str, 2048, "Volume %s already exists", volname);
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", err_str);
+                snprintf (err_str, sizeof (err_str), "Volume %s already exists",
+                          volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "brick count");
+                snprintf (err_str, sizeof (err_str), "Unable to get brick count"
+                          " for volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "type", &type);
-        /*if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get type");
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "type");
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get type of "
+                          "volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
-        }*/
+        }
 
         ret = dict_get_str (dict, "transport", &trans_type);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get transport-type");
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "transport-type");
+                snprintf (err_str, sizeof (err_str), "Unable to get "
+                          "transport-type of volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get bricks");
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "bricks");
+                snprintf (err_str, sizeof (err_str), "Unable to get bricks for "
+                          "volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if (!dict_get (dict, "force")) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get 'force' flag");
                 goto out;
         }
 
@@ -152,116 +146,82 @@ glusterd_handle_create_volume (rpcsvc_request_t *req)
         free_ptr = gf_strdup (uuid_utoa (volume_id));
         ret = dict_set_dynstr (dict, "volume-id", free_ptr);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "unable to set volume-id");
                 snprintf (err_str, sizeof (err_str), "Unable to set volume "
-                          "id");
+                          "id of volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
         free_ptr = NULL;
 
-        if (bricks) {
-                brick_list = gf_strdup (bricks);
-                free_ptr = brick_list;
-        }
-
-        gf_cmd_log ("Volume create", "on volname: %s type:%s count:%d bricks:%s",
-                    volname, ((type == 0)? "DEFAULT":
-                    ((type == 1)? "STRIPE":"REPLICATE")), brick_count, bricks);
-
-
-        while ( i < brick_count) {
-                i++;
-                brick= strtok_r (brick_list, " \n", &tmpptr);
-                brick_list = tmpptr;
-                ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str), "Unable to get "
-                                  "brick info from brick %s", brick);
-                        goto out;
-                }
-
-                ret = glusterd_new_brick_validate (brick, brickinfo, err_str,
-                                                   sizeof (err_str));
-                if (ret)
-                        goto out;
-                ret = glusterd_volume_brickinfo_get (brickinfo->uuid,
-                                                     brickinfo->hostname,
-                                                     brickinfo->path,
-                                                     &tmpvolinfo, &tmpbrkinfo,
-                                                     GF_PATH_PARTIAL);
-                if (!ret) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str), "Brick: %s:%s, %s"
-                                  " one of the bricks contain the other",
-                                  tmpbrkinfo->hostname, tmpbrkinfo->path, brick);
-                        goto out;
-                }
-                list_add_tail (&brickinfo->brick_list, &tmpvolinfo.bricks);
-                brickinfo = NULL;
-        }
-
         /* generate internal username and password */
 
         uuid_generate (tmp_uuid);
         username = gf_strdup (uuid_utoa (tmp_uuid));
         ret = dict_set_dynstr (dict, "internal-username", username);
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set username for "
+                        "volume %s", volname);
                 goto out;
+        }
 
         uuid_generate (tmp_uuid);
         password = gf_strdup (uuid_utoa (tmp_uuid));
         ret = dict_set_dynstr (dict, "internal-password", password);
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set password for "
+                        "volume %s", volname);
                 goto out;
+        }
 
-        ret = glusterd_op_begin (req, GD_OP_CREATE_VOLUME, dict);
-        gf_cmd_log ("Volume create", "on volname: %s %s", volname,
-                    (ret != 0) ? "FAILED": "SUCCESS");
+        ret = glusterd_op_begin_synctask (req, GD_OP_CREATE_VOLUME, dict);
 
 out:
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
                 rsp.op_ret = -1;
                 rsp.op_errno = 0;
                 if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str), "Operation failed");
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 rsp.op_errstr = err_str;
                 cli_rsp = &rsp;
-                glusterd_submit_reply(req, cli_rsp, NULL, 0, NULL,
-                                      (xdrproc_t)xdr_gf_cli_rsp);
-
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
                 ret = 0; //Client response sent, prevent second response
         }
 
-        if (free_ptr)
-                GF_FREE(free_ptr);
-
-        glusterd_volume_brickinfos_delete (&tmpvolinfo);
-        if (brickinfo)
-                glusterd_brickinfo_delete (brickinfo);
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        GF_FREE(free_ptr);
 
         return ret;
 }
 
 int
-glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+glusterd_handle_create_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_create_volume);
+}
+
+int
+__glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
-        char                            *dup_volname = NULL;
+        char                            *volname = NULL;
         dict_t                          *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_START_VOLUME;
+        char                            errstr[2048] = {0,};
+        xlator_t                        *this = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (errstr, sizeof (errstr), "Failed to decode message "
+                        "received from cli");
                 req->rpc_err = GARBAGE_ARGS;
+                gf_log (this->name, sizeof (errstr), "%s", errstr);
                 goto out;
         }
 
@@ -273,58 +233,68 @@ glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (errstr, sizeof (errstr), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
         }
 
-        ret = dict_get_str (dict, "volname", &dup_volname);
+        ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "dict get failed");
+                snprintf (errstr, sizeof (errstr), "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received start vol req"
-                "for volume %s", dup_volname);
-        ret = glusterd_op_begin (req, GD_OP_START_VOLUME, dict);
+        gf_log (this->name, GF_LOG_DEBUG, "Received start vol req"
+                " for volume %s", volname);
 
-        gf_cmd_log ("volume start","on volname: %s %s", dup_volname,
-                    ((ret == 0) ? "SUCCESS": "FAILED"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_START_VOLUME, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        free (cli_req.dict.dict_val); //its malloced by xdr
 
-        if (ret)
+        if (ret) {
+                if(errstr[0] == '\0')
+                        snprintf (errstr, sizeof (errstr), "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, errstr);
+        }
 
         return ret;
 }
 
+int
+glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_start_volume);
+}
 
 int
-glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
         char                            *dup_volname = NULL;
         dict_t                          *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_STOP_VOLUME;
+        xlator_t                        *this = NULL;
+        char                            err_str[2048] = {0,};
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode message "
+                          "received from cli");
                 req->rpc_err = GARBAGE_ARGS;
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
         if (cli_req.dict.dict_len) {
@@ -335,9 +305,11 @@ glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
         }
@@ -345,48 +317,59 @@ glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
         ret = dict_get_str (dict, "volname", &dup_volname);
 
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received stop vol req"
+        gf_log (this->name, GF_LOG_DEBUG, "Received stop vol req "
                 "for volume %s", dup_volname);
 
-        ret = glusterd_op_begin (req, GD_OP_STOP_VOLUME, dict);
-        gf_cmd_log ("Volume stop","on volname: %s %s", dup_volname,
-                    ((ret)?"FAILED":"SUCCESS"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_STOP_VOLUME, dict);
 
 out:
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        free (cli_req.dict.dict_val); //its malloced by xdr
 
         if (ret) {
-                if (dict)
-                        dict_unref (dict);
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
 
         return ret;
 }
 
 int
-glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_stop_volume);
+}
+
+int
+__glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
 {
         int32_t        ret         = -1;
         gf_cli_req     cli_req     = {{0,},};
         glusterd_op_t  cli_op      = GD_OP_DELETE_VOLUME;
         dict_t        *dict        = NULL;
         char          *volname     = NULL;
+        char          err_str[2048]= {0,};
+        xlator_t      *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
-                //failed to decode msg;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -399,65 +382,74 @@ glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
                         goto out;
                 }
         }
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
-        gf_cmd_log ("Volume delete","on volname: %s attempted", volname);
-
-        gf_log ("glusterd", GF_LOG_INFO, "Received delete vol req"
+        gf_log (this->name, GF_LOG_DEBUG, "Received delete vol req"
                 "for volume %s", volname);
 
-        ret = glusterd_op_begin (req, GD_OP_DELETE_VOLUME, dict);
-        gf_cmd_log ("Volume delete", "on volname: %s %s", volname,
-                   ((ret) ? "FAILED" : "SUCCESS"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_DELETE_VOLUME, dict);
 
 out:
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val); //its malloced by xdr
-        if (ret && dict)
-                dict_unref (dict);
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        free (cli_req.dict.dict_val); //its malloced by xdr
 
         if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, err_str);
         }
 
         return ret;
 }
 
 int
-glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_delete_volume);
+}
+
+int
+__glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
         dict_t                          *dict = NULL;
         glusterd_op_t                   cli_op = GD_OP_HEAL_VOLUME;
         char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        xlator_t                        *this = NULL;
+        char                            op_errstr[2048] = {0,};
 
         GF_ASSERT (req);
 
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
 
+        this = THIS;
+        GF_ASSERT (this);
+
         if (cli_req.dict.dict_len) {
                 /* Unserialize the dictionary */
                 dict  = dict_new ();
@@ -466,9 +458,11 @@ glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (op_errstr, sizeof (op_errstr),
+                        "Unable to decode the command");
                         goto out;
                 } else {
                         dict->extra_stdfree = cli_req.dict.dict_val;
@@ -477,33 +471,54 @@ glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (op_errstr, sizeof (op_errstr), "Unable to find "
+                          "volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
                 goto out;
         }
-        gf_log ("glusterd", GF_LOG_INFO, "Received heal vol req"
+
+        gf_log (this->name, GF_LOG_INFO, "Received heal vol req "
                 "for volume %s", volname);
 
-        ret = glusterd_op_begin (req, GD_OP_HEAL_VOLUME, dict);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (op_errstr, sizeof (op_errstr),
+                          "Volume %s does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                goto out;
+        }
 
-        gf_cmd_log ("volume heal","on volname: %s %s", volname,
-                    ((ret == 0) ? "SUCCESS": "FAILED"));
+        ret = glusterd_add_bricks_hname_path_to_dict (dict, volinfo);
+        if (ret)
+                goto out;
 
-out:
-        if (ret && dict)
-                dict_unref (dict);
+        ret = dict_set_int32 (dict, "count", volinfo->brick_count);
+        if (ret)
+                goto out;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        ret = glusterd_op_begin_synctask (req, GD_OP_HEAL_VOLUME, dict);
 
-        if (ret)
+out:
+        if (ret) {
+                if (op_errstr[0] == '\0')
+                        snprintf (op_errstr, sizeof (op_errstr),
+                                  "operation failed");
                 ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     NULL, "operation failed");
+                                                     dict, op_errstr);
+        }
 
         return ret;
 }
 
 int
-glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_heal_volume);
+}
+
+int
+__glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
         gf_cli_req                      cli_req = {{0,}};
@@ -511,12 +526,21 @@ glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
         char                            *options = NULL;
         dict_t                          *dict = NULL;
         int32_t                         option_cnt = 0;
+        glusterd_op_t                   cli_op = GD_OP_STATEDUMP_VOLUME;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
 
         GF_ASSERT (req);
 
         ret = -1;
-        if (!xdr_to_generic (req->msg[0], &cli_req,
-                             (xdrproc_t)xdr_gf_cli_req)) {
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
@@ -528,50 +552,166 @@ glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
                                         cli_req.dict.dict_len,
                                         &dict);
                 if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_ERROR,
                                 "failed to "
                                 "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to "
+                                  "decode the command");
                         goto out;
                 }
         }
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get volname");
+                snprintf (err_str, sizeof (err_str), "Unable to get the volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_str (dict, "options", &options);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get options");
+                snprintf (err_str, sizeof (err_str), "Unable to get options");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "option_cnt", &option_cnt);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get option cnt");
+                snprintf (err_str , sizeof (err_str), "Unable to get option "
+                          "count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
                 goto out;
         }
 
+        if (priv->op_version == GD_OP_VERSION_MIN &&
+            strstr (options, "quotad")) {
+                snprintf (err_str, sizeof (err_str), "The cluster is operating "
+                          "at op-version 1. Taking quotad's statedump is "
+                          "disallowed in this state");
+                ret = -1;
+                goto out;
+        }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Recieved statedump request for "
+        gf_log (this->name, GF_LOG_INFO, "Received statedump request for "
                 "volume %s with options %s", volname, options);
 
-        ret = glusterd_op_begin (req, GD_OP_STATEDUMP_VOLUME, dict);
-
-        gf_cmd_log ("statedump", "on volume %s %s", volname,
-                    ((0 == ret) ? "SUCCEEDED" : "FAILED"));
+        ret = glusterd_op_begin_synctask (req, GD_OP_STATEDUMP_VOLUME, dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
-        if (cli_req.dict.dict_val)
-                free (cli_req.dict.dict_val);
-        glusterd_friend_sm ();
-        glusterd_op_sm();
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
 
         return ret;
 }
 
+int
+glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_statedump_volume);
+}
+
+#ifdef HAVE_BD_XLATOR
+/*
+ * Validates if given VG in the brick exists or not. Also checks if VG has
+ * GF_XATTR_VOL_ID_KEY tag set to avoid using same VG for multiple bricks.
+ * Tag is checked only during glusterd_op_stage_create_volume. Tag is set during
+ * glusterd_validate_and_create_brickpath().
+ * @brick - brick info, @check_tag - check for VG tag or not
+ * @msg - Error message to return to caller
+ */
+int
+glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg)
+{
+        lvm_t                     handle     = NULL;
+        vg_t                      vg         = NULL;
+        char                     *vg_name    = NULL;
+        int                       retval     = 0;
+        char                     *p          = NULL;
+        char                     *ptr        = NULL;
+        struct dm_list           *dm_lvlist = NULL;
+        struct dm_list           *dm_seglist = NULL;
+        struct lvm_lv_list       *lv_list    = NULL;
+        struct lvm_property_value prop       = {0, };
+        struct lvm_lvseg_list    *seglist    = NULL;
+        struct dm_list           *taglist    = NULL;
+        struct lvm_str_list      *strl       = NULL;
+
+        handle = lvm_init (NULL);
+        if (!handle) {
+                sprintf (msg, "lvm_init failed, could not validate vg");
+                return -1;
+        }
+        if (*brick->vg == '\0') { /* BD xlator has vg in brick->path */
+                p = gf_strdup (brick->path);
+                vg_name = strtok_r (p, "/", &ptr);
+        } else
+                vg_name = brick->vg;
+
+        vg = lvm_vg_open (handle, vg_name, "r", 0);
+        if (!vg) {
+                sprintf (msg, "no such vg: %s", vg_name);
+                retval = -1;
+                goto out;
+        }
+        if (!check_tag)
+                goto next;
+
+        taglist = lvm_vg_get_tags (vg);
+        if (!taglist)
+                goto next;
+
+        dm_list_iterate_items (strl, taglist) {
+                if (!strncmp(strl->str, GF_XATTR_VOL_ID_KEY,
+                             strlen (GF_XATTR_VOL_ID_KEY))) {
+                            sprintf (msg, "VG %s is already part of"
+                                    " a brick", vg_name);
+                            retval = -1;
+                            goto out;
+                    }
+        }
+next:
+
+        brick->caps = CAPS_BD | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+
+        dm_lvlist = lvm_vg_list_lvs (vg);
+        if (!dm_lvlist)
+                goto out;
+
+        dm_list_iterate_items (lv_list, dm_lvlist) {
+                dm_seglist = lvm_lv_list_lvsegs (lv_list->lv);
+                dm_list_iterate_items (seglist, dm_seglist) {
+                        prop = lvm_lvseg_get_property (seglist->lvseg,
+                                                       "segtype");
+                        if (!prop.is_valid || !prop.value.string)
+                                continue;
+                        if (!strcmp (prop.value.string, "thin-pool")) {
+                                brick->caps |= CAPS_THIN;
+                                gf_log (THIS->name, GF_LOG_INFO, "Thin Pool "
+                                        "\"%s\" will be used for thin LVs",
+                                        lvm_lv_get_name (lv_list->lv));
+                                break;
+                        }
+                }
+        }
+
+        retval = 0;
+out:
+        if (vg)
+                lvm_vg_close (vg);
+        lvm_quit (handle);
+        if (p)
+                GF_FREE (p);
+        return retval;
+}
+#endif
+
 /* op-sm */
 int
 glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
@@ -587,73 +727,68 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
         int32_t                                 i = 0;
         char                                    *brick = NULL;
         char                                    *tmpptr = NULL;
-        char                                    cmd_str[1024];
         xlator_t                                *this = NULL;
         glusterd_conf_t                         *priv = NULL;
         char                                    msg[2048] = {0};
         uuid_t                                  volume_uuid;
         char                                    *volume_uuid_str;
+        gf_boolean_t                             is_force = _gf_false;
 
         this = THIS;
-        if (!this) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "this is NULL");
-                goto out;
-        }
-
+        GF_ASSERT (this);
         priv = this->private;
-        if (!priv) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "priv is NULL");
-                goto out;
-        }
+        GF_ASSERT (priv);
 
         ret = dict_get_str (dict, "volname", &volname);
-
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         exists = glusterd_check_volume_exists (volname);
-
         if (exists) {
                 snprintf (msg, sizeof (msg), "Volume %s already exists",
                           volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
         } else {
                 ret = 0;
         }
+
         ret = dict_get_int32 (dict, "count", &brick_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count "
+                        "for volume %s", volname);
                 goto out;
         }
+
         ret = dict_get_str (dict, "volume-id", &volume_uuid_str);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume id");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume id of "
+                        "volume %s", volname);
                 goto out;
         }
+
         ret = uuid_parse (volume_uuid_str, volume_uuid);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to parse volume id");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to parse volume id of"
+                        " volume %s", volname);
                 goto out;
         }
 
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get bricks");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
+                        "volume %s", volname);
                 goto out;
         }
 
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
         if (bricks) {
                 brick_list = gf_strdup (bricks);
                 if (!brick_list) {
                         ret = -1;
-                        gf_log ("", GF_LOG_ERROR, "Out of memory");
                         goto out;
                 } else {
                         free_ptr = brick_list;
@@ -669,30 +804,37 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
                         !glusterd_is_valid_volfpath (volname, brick)) {
                         snprintf (msg, sizeof (msg), "brick path %s is too "
                                   "long.", brick);
-                        gf_log ("", GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-
                         ret = -1;
                         goto out;
                 }
 
-                ret = glusterd_brickinfo_from_brick (brick, &brick_info);
+                ret = glusterd_brickinfo_new_from_brick (brick, &brick_info);
                 if (ret)
                         goto out;
-                snprintf (cmd_str, 1024, "%s", brick_info->path);
+
+                ret = glusterd_new_brick_validate (brick, brick_info, msg,
+                                                   sizeof (msg));
+                if (ret)
+                        goto out;
+
                 ret = glusterd_resolve_brick (brick_info);
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "cannot resolve "
-                                "brick: %s:%s", brick_info->hostname,
-                                brick_info->path);
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brick_info->hostname, brick_info->path);
                         goto out;
                 }
 
-                if (!uuid_compare (brick_info->uuid, priv->uuid)) {
-                        ret = glusterd_brick_create_path (brick_info->hostname,
-                                                          brick_info->path,
-                                                          volume_uuid,
-                                                          0777, op_errstr);
+                if (!uuid_compare (brick_info->uuid, MY_UUID)) {
+#ifdef HAVE_BD_XLATOR
+                        if (brick_info->vg[0]) {
+                                ret = glusterd_is_valid_vg (brick_info, 1, msg);
+                                if (ret)
+                                        goto out;
+                        }
+#endif
+                        ret = glusterd_validate_and_create_brickpath (brick_info,
+                                                          volume_uuid, op_errstr,
+                                                          is_force);
                         if (ret)
                                 goto out;
                         brick_list = tmpptr;
@@ -701,11 +843,15 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
                 brick_info = NULL;
         }
 out:
-        if (free_ptr)
-                GF_FREE (free_ptr);
+        GF_FREE (free_ptr);
         if (brick_info)
                 glusterd_brickinfo_delete (brick_info);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -714,19 +860,23 @@ int
 glusterd_op_stop_volume_args_get (dict_t *dict, char** volname, int *flags)
 {
         int ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         if (!dict || !volname || !flags)
                 goto out;
 
         ret = dict_get_str (dict, "volname", volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "flags", flags);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get flags");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get flags");
                 goto out;
         }
 out:
@@ -773,16 +923,18 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
         gf_boolean_t                            exists = _gf_false;
         glusterd_volinfo_t                      *volinfo = NULL;
         glusterd_brickinfo_t                    *brickinfo = NULL;
-        char                                    msg[2048];
+        char                                    msg[2048] = {0,};
         glusterd_conf_t                         *priv = NULL;
+        xlator_t                                *this = NULL;
+        uuid_t                                  volume_id = {0,};
+        char                                    volid[50] = {0,};
+        char                                    xattr_volid[50] = {0,};
+        int                                     caps = 0;
 
-        priv = THIS->private;
-        if (!priv) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "priv is NULL");
-                ret = -1;
-                goto out;
-        }
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
 
         ret = glusterd_op_start_volume_args_get (dict, &volname, &flags);
         if (ret)
@@ -791,53 +943,110 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
         exists = glusterd_check_volume_exists (volname);
 
         if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist", volname);
-                gf_log ("", GF_LOG_ERROR, "%s",
-                        msg);
-                *op_errstr = gf_strdup (msg);
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
                 ret = -1;
-        } else {
-                ret = 0;
+                goto out;
         }
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
         if (ret)
                 goto out;
 
+        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+                if (glusterd_is_volume_started (volinfo)) {
+                        snprintf (msg, sizeof (msg), "Volume %s already "
+                                  "started", volname);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 ret = glusterd_resolve_brick (brickinfo);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to resolve brick %s:%s",
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
                                 brickinfo->hostname, brickinfo->path);
                         goto out;
                 }
 
-                if (!uuid_compare (brickinfo->uuid, priv->uuid)) {
-                        ret = glusterd_brick_create_path (brickinfo->hostname,
-                                                          brickinfo->path,
-                                                          volinfo->volume_id,
-                                                          0777, op_errstr);
-                        if (ret)
-                                goto out;
-                }
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
 
-                if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                        if (glusterd_is_volume_started (volinfo)) {
-                                snprintf (msg, sizeof (msg), "Volume %s already"
-                                          " started", volname);
-                                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
-                                *op_errstr = gf_strdup (msg);
-                                ret = -1;
+                ret = gf_lstat_dir (brickinfo->path, NULL);
+                if (ret && (flags & GF_CLI_FLAG_OP_FORCE)) {
+                        continue;
+                } else if (ret) {
+                        snprintf (msg, sizeof (msg), "Failed to find "
+                                          "brick directory %s for volume %s. "
+                                          "Reason : %s", brickinfo->path,
+                                          volname, strerror (errno));
+                        goto out;
+                }
+                ret = sys_lgetxattr (brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                                     volume_id, 16);
+                if (ret < 0 && (!(flags & GF_CLI_FLAG_OP_FORCE))) {
+                        snprintf (msg, sizeof (msg), "Failed to get "
+                                  "extended attribute %s for brick dir %s. "
+                                  "Reason : %s", GF_XATTR_VOL_ID_KEY,
+                                  brickinfo->path, strerror (errno));
+                        ret = -1;
+                        goto out;
+                } else if (ret < 0) {
+                        ret = sys_lsetxattr (brickinfo->path,
+                                             GF_XATTR_VOL_ID_KEY,
+                                             volinfo->volume_id, 16,
+                                             XATTR_CREATE);
+                        if (ret) {
+                                snprintf (msg, sizeof (msg), "Failed to set "
+                                        "extended attribute %s on %s. Reason: "
+                                        "%s", GF_XATTR_VOL_ID_KEY,
+                                        brickinfo->path, strerror (errno));
                                 goto out;
+                        } else {
+                                continue;
                         }
                 }
+                if (uuid_compare (volinfo->volume_id, volume_id)) {
+                        snprintf (msg, sizeof (msg), "Volume id mismatch for "
+                                  "brick %s:%s. Expected volume id %s, "
+                                  "volume id %s found", brickinfo->hostname,
+                                  brickinfo->path,
+                                  uuid_utoa_r (volinfo->volume_id, volid),
+                                  uuid_utoa_r (volume_id, xattr_volid));
+                        ret = -1;
+                        goto out;
+                }
+#ifdef HAVE_BD_XLATOR
+                if (brickinfo->vg[0])
+                        caps = CAPS_BD | CAPS_THIN |
+                                CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+                /* Check for VG/thin pool if its BD volume */
+                if (brickinfo->vg[0]) {
+                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                        if (ret)
+                                goto out;
+                        /* if anyone of the brick does not have thin support,
+                           disable it for entire volume */
+                        caps &= brickinfo->caps;
+                } else
+                        caps = 0;
+#endif
         }
 
+        volinfo->caps = caps;
         ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-
+        if (ret && (msg[0] != '\0')) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
         return ret;
 }
 
@@ -851,7 +1060,11 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         gf_boolean_t                            is_run = _gf_false;
         glusterd_volinfo_t                      *volinfo = NULL;
         char                                    msg[2048] = {0};
+        xlator_t                                *this = NULL;
+        gsync_status_param_t                    param = {0,};
 
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
         if (ret)
@@ -860,16 +1073,20 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         exists = glusterd_check_volume_exists (volname);
 
         if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist", volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 ret = -1;
                 goto out;
-        } else {
-                ret = 0;
         }
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
         if (ret)
                 goto out;
 
@@ -880,25 +1097,37 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         if (_gf_false == glusterd_is_volume_started (volinfo)) {
                 snprintf (msg, sizeof(msg), "Volume %s "
                           "is not in the started state", volname);
-                gf_log ("", GF_LOG_ERROR, "Volume %s "
-                        "has not been started", volname);
-                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 ret = -1;
                 goto out;
         }
         ret = glusterd_check_gsync_running (volinfo, &is_run);
         if (ret && (is_run == _gf_false))
-                gf_log ("", GF_LOG_WARNING, "Unable to get the status"
+                gf_log (this->name, GF_LOG_WARNING, "Unable to get the status"
                         " of active "GEOREP" session");
-        if (is_run) {
-                gf_log ("", GF_LOG_WARNING, GEOREP" sessions active"
+
+        param.volinfo = volinfo;
+        ret = dict_foreach (volinfo->gsync_slaves, _get_slave_status, &param);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "_get_slave_satus failed");
+                snprintf (msg, sizeof(msg), GEOREP" Unable to get the status "
+                          "of active "GEOREP" session for the volume '%s'.\n"
+                          "Please check the log file for more info. Use "
+                          "'force' option to ignore and stop the volume.",
+                          volname);
+                ret = -1;
+                goto out;
+        }
+
+        if (is_run && param.is_active) {
+                gf_log (this->name, GF_LOG_WARNING, GEOREP" sessions active"
                         "for the volume %s ", volname);
                 snprintf (msg, sizeof(msg), GEOREP" sessions are active "
                           "for the volume '%s'.\nUse 'volume "GEOREP" "
-                          "status' command for more info. Use 'force'"
-                          "option to ignore and stop stop the volume",
+                          "status' command for more info. Use 'force' "
+                          "option to ignore and stop the volume.",
                           volname);
-                *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
         }
@@ -907,8 +1136,8 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
                 snprintf (msg, sizeof (msg), "Replace brick is in progress on "
                           "volume %s. Please retry after replace-brick "
                           "operation is committed or aborted", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_WARNING, "replace-brick in progress "
+                        "on volume %s", volname);
                 ret = -1;
                 goto out;
         }
@@ -916,23 +1145,22 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         if (glusterd_is_defrag_on (volinfo)) {
                 snprintf (msg, sizeof(msg), "rebalance session is "
                           "in progress for the volume '%s'", volname);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
                 ret = -1;
                 goto out;
         }
-        if (volinfo->rb_status != GF_RB_STATUS_NONE) {
+        if (volinfo->rep_brick.rb_status != GF_RB_STATUS_NONE) {
                 snprintf (msg, sizeof(msg), "replace-brick session is "
                           "in progress for the volume '%s'", volname);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
                 ret = -1;
                 goto out;
         }
 
 out:
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (msg[0] != 0)
+                *op_errstr = gf_strdup (msg);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -945,21 +1173,20 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
         gf_boolean_t                            exists = _gf_false;
         glusterd_volinfo_t                      *volinfo = NULL;
         char                                    msg[2048] = {0};
+        xlator_t                                *this = NULL;
 
-        ret = dict_get_str (dict, "volname", &volname);
+        this = THIS;
+        GF_ASSERT (this);
 
+        ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         exists = glusterd_check_volume_exists (volname);
-
         if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
                 ret = -1;
                 goto out;
         } else {
@@ -967,7 +1194,12 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
         }
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                goto out;
+        }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
         if (ret)
                 goto out;
 
@@ -975,8 +1207,6 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
                 snprintf (msg, sizeof (msg), "Volume %s has been started."
                           "Volume needs to be stopped before deletion.",
                           volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
         }
@@ -984,7 +1214,11 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
         ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
 
         return ret;
 }
@@ -1000,11 +1234,13 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
         glusterd_conf_t                         *priv = NULL;
         dict_t                                  *opt_dict = NULL;
         gf_xl_afr_op_t                          heal_op = GF_AFR_OP_INVALID;
+        xlator_t                                *this = NULL;
 
-        priv = THIS->private;
+        this = THIS;
+        priv = this->private;
         if (!priv) {
                 ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_ERROR,
                         "priv is NULL");
                 goto out;
         }
@@ -1019,17 +1255,21 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
         if (ret) {
                 ret = -1;
                 snprintf (msg, sizeof (msg), "Volume %s does not exist", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
                 *op_errstr = gf_strdup (msg);
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         if (!glusterd_is_volume_replicate (volinfo)) {
                 ret = -1;
-                snprintf (msg, sizeof (msg), "Volume %s is not a replicate "
-                          "type volume", volname);
+                snprintf (msg, sizeof (msg), "Volume %s is not of type "
+                          "replicate", volname);
                 *op_errstr = gf_strdup (msg);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
                 goto out;
         }
 
@@ -1055,29 +1295,37 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
                 snprintf (msg, sizeof (msg), "Self-heal-daemon is "
                           "disabled. Heal will not be triggered on volume %s",
                           volname);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (!glusterd_shd_is_running ()) {
-                ret = -1;
-                snprintf (msg, sizeof (msg), "Self-heal daemon is not "
-                          "running. Check self-heal daemon log file.");
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
                 *op_errstr = gf_strdup (msg);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
         if (ret || (heal_op == GF_AFR_OP_INVALID)) {
                 ret = -1;
-                snprintf (msg, sizeof (msg), "Invalid heal-op");
-                *op_errstr = gf_strdup (msg);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
+                *op_errstr = gf_strdup("Invalid heal-op");
+                gf_log (this->name, GF_LOG_WARNING, "%s", "Invalid heal-op");
                 goto out;
         }
 
+        switch (heal_op) {
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                        break;
+                default:
+                        if (!glusterd_is_nodesvc_online("glustershd")){
+                                ret = -1;
+                                *op_errstr = gf_strdup ("Self-heal daemon is "
+                                                "not running. Check self-heal "
+                                                "daemon log file.");
+                                gf_log (this->name, GF_LOG_WARNING, "%s",
+                                        "Self-heal daemon is not running."
+                                        "Check self-heal daemon log file.");
+                                goto out;
+                        }
+        }
+
         ret = 0;
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
@@ -1095,6 +1343,13 @@ glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr)
         gf_boolean_t            is_running = _gf_false;
         glusterd_volinfo_t      *volinfo = NULL;
         char                    msg[2408] = {0,};
+        xlator_t                *this     = NULL;
+        glusterd_conf_t         *priv     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
 
         ret = glusterd_op_statedump_volume_args_get (dict, &volname, &options,
                                                      &option_cnt);
@@ -1103,25 +1358,41 @@ glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr)
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
-                snprintf (msg, sizeof(msg), "Volume %s does not exist",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+                snprintf (msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         is_running = glusterd_is_volume_started (volinfo);
         if (!is_running) {
-                snprintf (msg, sizeof(msg), "Volume %s is not in a started"
+                snprintf (msg, sizeof(msg), "Volume %s is not in the started"
                           " state", volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
                 ret = -1;
                 goto out;
         }
 
+        if (priv->op_version == GD_OP_VERSION_MIN &&
+            strstr (options, "quotad")) {
+                snprintf (msg, sizeof (msg), "The cluster is operating "
+                          "at op-version 1. Taking quotad's statedump is "
+                          "disallowed in this state");
+                ret = -1;
+                goto out;
+        }
+        if ((strstr (options, "quotad")) &&
+            (!glusterd_is_volume_quota_enabled (volinfo))) {
+                    snprintf (msg, sizeof (msg), "Quota is not enabled on "
+                              "volume %s", volname);
+                    ret = -1;
+                    goto out;
+        }
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        if (ret && msg[0] != '\0')
+                *op_errstr = gf_strdup (msg);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
 
@@ -1177,6 +1448,10 @@ glusterd_op_stage_clearlocks_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
         if (!glusterd_is_volume_started (volinfo)) {
                 snprintf (msg, sizeof(msg), "Volume %s is not started",
                           volname);
@@ -1191,7 +1466,6 @@ out:
         return ret;
 }
 
-
 int
 glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 {
@@ -1213,6 +1487,9 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         char                 *str        = NULL;
         char                 *username   = NULL;
         char                 *password   = NULL;
+        int                   caps       = 0;
+        int                   brickid    = 0;
+        char                  msg[1024] __attribute__((unused)) = {0, };
 
         this = THIS;
         GF_ASSERT (this);
@@ -1224,15 +1501,14 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to allocate memory");
+                        "Unable to allocate memory for volinfo");
                 goto out;
         }
 
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
@@ -1241,13 +1517,15 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         ret = dict_get_int32 (dict, "type", &volinfo->type);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get type");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of volume"
+                        " %s", volname);
                 goto out;
         }
 
         ret = dict_get_int32 (dict, "count", &volinfo->brick_count);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get count");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count of"
+                        " volume %s", volname);
                 goto out;
         }
 
@@ -1261,7 +1539,8 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
+                        "volume %s", volname);
                 goto out;
         }
 
@@ -1273,28 +1552,44 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
                 ret = dict_get_int32 (dict, "replica-count",
                                       &volinfo->replica_count);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                 "replica count for volume %s", volname);
                         goto out;
+                }
         } else if (GF_CLUSTER_TYPE_STRIPE == volinfo->type) {
                 ret = dict_get_int32 (dict, "stripe-count",
                                       &volinfo->stripe_count);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get stripe"
+                                " count for volume %s", volname);
                         goto out;
+                }
         } else if (GF_CLUSTER_TYPE_STRIPE_REPLICATE == volinfo->type) {
                 ret = dict_get_int32 (dict, "stripe-count",
                                       &volinfo->stripe_count);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get stripe"
+                                " count for volume %s", volname);
                         goto out;
+                }
                 ret = dict_get_int32 (dict, "replica-count",
                                       &volinfo->replica_count);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "replica count for volume %s", volname);
                         goto out;
+                }
         }
 
         /* dist-leaf-count is the count of brick nodes for a given
            subvolume of distribute */
-        volinfo->dist_leaf_count = (volinfo->stripe_count *
-                                    volinfo->replica_count);
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+
+        /* subvol_count is the count of number of subvolumes present
+           for a given distribute volume */
+        volinfo->subvol_count = (volinfo->brick_count /
+                                 volinfo->dist_leaf_count);
 
         /* Keep sub-count same as earlier, for the sake of backward
            compatibility */
@@ -1304,27 +1599,28 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         ret = dict_get_str (dict, "transport", &trans_type);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get transport");
+                        "Unable to get transport type of volume %s", volname);
                 goto out;
         }
 
         ret = dict_get_str (dict, "volume-id", &str);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get volume-id");
+                        "Unable to get volume-id of volume %s", volname);
                 goto out;
         }
         ret = uuid_parse (str, volinfo->volume_id);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "unable to parse uuid %s", str);
+                        "unable to parse uuid %s of volume %s", str, volname);
                 goto out;
         }
 
         ret = dict_get_str (dict, "internal-username", &username);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "unable to get internal username");
+                        "unable to get internal username of volume %s",
+                        volname);
                 goto out;
         }
         glusterd_auth_set_username (volinfo, username);
@@ -1332,7 +1628,8 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         ret = dict_get_str (dict, "internal-password", &password);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "unable to get internal password");
+                        "unable to get internal password of volume %s",
+                        volname);
                 goto out;
         }
         glusterd_auth_set_password (volinfo, password);
@@ -1355,20 +1652,55 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         if (count)
                 brick = strtok_r (brick_list+1, " \n", &saveptr);
+        caps = CAPS_BD | CAPS_THIN | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
 
+        brickid = glusterd_get_next_available_brickid (volinfo);
+        if (brickid < 0)
+                goto out;
         while ( i <= count) {
-                ret = glusterd_brickinfo_from_brick (brick, &brickinfo);
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
                 if (ret)
                         goto out;
 
+                GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo,
+                                                      brickid++);
+
                 ret = glusterd_resolve_brick (brickinfo);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brickinfo->hostname, brickinfo->path);
                         goto out;
+                }
+
+#ifdef HAVE_BD_XLATOR
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                                msg);
+                                        goto out;
+                                }
+
+                                /* if anyone of the brick does not have thin
+                                   support, disable it for entire volume */
+                                caps &= brickinfo->caps;
+
+
+                        } else
+                                caps = 0;
+                }
+#endif
+
                 list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
                 brick = strtok_r (NULL, " \n", &saveptr);
                 i++;
         }
 
+        gd_update_volume_op_versions (volinfo);
+
+        volinfo->caps = caps;
+
         ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret) {
                 glusterd_store_delete_volume (volinfo);
@@ -1382,20 +1714,15 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        ret = glusterd_volume_compute_cksum (volinfo);
-        if (ret) {
-                *op_errstr = gf_strdup ("Failed to compute checksum of volume");
-                goto out;
-        }
-
-        volinfo->defrag_status = 0;
-        list_add_tail (&volinfo->vol_list, &priv->volumes);
+        volinfo->rebal.defrag_status = 0;
+        list_add_order (&volinfo->vol_list, &priv->volumes,
+                         glusterd_compare_volume_name);
         vol_added = _gf_true;
+
 out:
-        if (free_ptr)
-                GF_FREE(free_ptr);
+        GF_FREE(free_ptr);
         if (!vol_added && volinfo)
-                glusterd_volinfo_delete (volinfo);
+                glusterd_volinfo_unref (volinfo);
         return ret;
 }
 
@@ -1407,18 +1734,28 @@ glusterd_op_start_volume (dict_t *dict, char **op_errstr)
         int                                     flags = 0;
         glusterd_volinfo_t                      *volinfo = NULL;
         glusterd_brickinfo_t                    *brickinfo = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = glusterd_op_start_volume_args_get (dict, &volname, &flags);
         if (ret)
                 goto out;
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
+        }
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_start (volinfo, brickinfo);
-                if (ret)
+                ret = glusterd_brick_start (volinfo, brickinfo, _gf_true);
+                /* If 'force' try to start all bricks regardless of success or
+                 * failure
+                 */
+                if (!(flags & GF_CLI_FLAG_OP_FORCE) && ret)
                         goto out;
         }
 
@@ -1431,7 +1768,7 @@ glusterd_op_start_volume (dict_t *dict, char **op_errstr)
         ret = glusterd_nodesvcs_handle_graph_change (volinfo);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d ", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
         return ret;
 }
 
@@ -1444,18 +1781,27 @@ glusterd_op_stop_volume (dict_t *dict)
         char                                    *volname = NULL;
         glusterd_volinfo_t                      *volinfo = NULL;
         glusterd_brickinfo_t                    *brickinfo = NULL;
+        xlator_t                                *this = NULL;
+        char                                    mountdir[PATH_MAX] = {0,};
+        runner_t                                runner = {0,};
+        char                                    pidfile[PATH_MAX] = {0,};
+
+        this = THIS;
+        GF_ASSERT (this);
 
         ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
         if (ret)
                 goto out;
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
                 goto out;
+        }
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_stop (volinfo, brickinfo);
+                ret = glusterd_brick_stop (volinfo, brickinfo, _gf_false);
                 if (ret)
                         goto out;
         }
@@ -1466,6 +1812,30 @@ glusterd_op_stop_volume (dict_t *dict)
         if (ret)
                 goto out;
 
+        /* If quota auxiliary mount is present, unmount it */
+        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile, volname);
+
+        if (!gf_is_service_running (pidfile, NULL)) {
+                gf_log (this->name, GF_LOG_DEBUG, "Aux mount of volume %s "
+                        "absent", volname);
+        } else {
+                GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
+
+                runinit (&runner);
+                runner_add_args (&runner, "umount",
+
+                #if GF_LINUX_HOST_OS
+                                "-l",
+                #endif
+                                mountdir, NULL);
+                ret = runner_run_reuse (&runner);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "umount on %s failed, "
+                                "reason : %s", mountdir, strerror (errno));
+
+                runner_end (&runner);
+        }
+
         ret = glusterd_nodesvcs_handle_graph_change (volinfo);
 out:
         return ret;
@@ -1487,18 +1857,24 @@ glusterd_op_delete_volume (dict_t *dict)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
                 goto out;
         }
 
         ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
 
+        ret = glusterd_remove_auxiliary_mount (volname);
         if (ret)
                 goto out;
 
         ret = glusterd_delete_volume (volinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
         return ret;
 }
 
@@ -1534,14 +1910,26 @@ glusterd_op_statedump_volume (dict_t *dict, char **op_errstr)
                 ret = glusterd_nfs_statedump (options, option_cnt, op_errstr);
                 if (ret)
                         goto out;
+
+        } else if (strstr (options, "quotad")) {
+                ret = glusterd_quotad_statedump (options, option_cnt,
+                                                 op_errstr);
+                if (ret)
+                        goto out;
         } else {
                 list_for_each_entry (brickinfo, &volinfo->bricks,
                                                         brick_list) {
                         ret = glusterd_brick_statedump (volinfo, brickinfo,
                                                         options, option_cnt,
                                                         op_errstr);
+                        /* Let us take the statedump of other bricks instead of
+                         * exiting, if statedump of this brick fails.
+                         */
                         if (ret)
-                                goto out;
+                                gf_log (THIS->name, GF_LOG_WARNING, "could not "
+                                        "take the statedump of the brick %s:%s."
+                                        " Proceeding to other bricks",
+                                        brickinfo->hostname, brickinfo->path);
                 }
         }
 
@@ -1610,7 +1998,9 @@ glusterd_clearlocks_unmount (glusterd_volinfo_t *volinfo, char *mntpt)
         runner_add_args (&runner, "/bin/umount", "-f", NULL);
         runner_argprintf (&runner, "%s", mntpt);
 
+        synclock_unlock (&priv->big_lock);
         ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret) {
                 ret = 0;
                 gf_log ("", GF_LOG_DEBUG,
@@ -1654,6 +2044,9 @@ glusterd_clearlocks_mount (glusterd_volinfo_t *volinfo, char **xl_opts,
         glusterd_conf_t *priv                           = NULL;
         runner_t        runner                          = {0,};
         char            client_volfpath[PATH_MAX]       = {0,};
+        char            self_heal_opts[3][1024]      = {"*replicate*.data-self-heal=off",
+                                                        "*replicate*.metadata-self-heal=off",
+                                                        "*replicate*.entry-self-heal=off"};
 
         priv = THIS->private;
 
@@ -1665,14 +2058,23 @@ glusterd_clearlocks_mount (glusterd_volinfo_t *volinfo, char **xl_opts,
         runner_add_arg (&runner, "-l");
         runner_argprintf (&runner, DEFAULT_LOG_FILE_DIRECTORY
                           "/%s-clearlocks-mnt.log", volinfo->volname);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
 
         for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++) {
                 runner_add_arg (&runner, "--xlator-option");
                 runner_argprintf (&runner, "%s", xl_opts[i]);
         }
 
+        for (i = 0; i < 3; i++) {
+                runner_add_args (&runner, "--xlator-option",
+                                 self_heal_opts[i], NULL);
+        }
+
         runner_argprintf (&runner, "%s", mntpt);
+        synclock_unlock (&priv->big_lock);
         ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_DEBUG,
                         "Could not start glusterfs");
@@ -1708,7 +2110,7 @@ glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
         index = -1;
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 index++;
-                if (uuid_compare (brickinfo->uuid, priv->uuid))
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
                         continue;
 
                 port = pmap_registry_search (THIS, brickinfo->path,
@@ -1736,7 +2138,7 @@ out:
 }
 
 int
-glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr)
+glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
         int32_t                         ret                 = -1;
         int                             i                   = 0;
@@ -1751,7 +2153,6 @@ glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr)
         char                            result[PATH_MAX]    = {0,};
         char                            *mntpt              = NULL;
         char                            **xl_opts           = NULL;
-        dict_t                          *ctx                = NULL;
         glusterd_volinfo_t              *volinfo            = NULL;
 
         ret = dict_get_str (dict, "volname", &volname);
@@ -1840,14 +2241,8 @@ glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr)
                 goto umount;
         }
 
-        ctx = glusterd_op_get_ctx ();
-        if (!ctx)
-                /*Impossible. Only originator glusterd can
-                 * come here. */
-                goto umount;
-
         free_ptr = gf_strdup(result);
-        if (dict_set_dynstr (ctx, "lk-summary", free_ptr)) {
+        if (dict_set_dynstr (rsp_dict, "lk-summary", free_ptr)) {
                 GF_FREE (free_ptr);
                 snprintf (msg, sizeof (msg), "Failed to set clear-locks "
                           "result");
@@ -1871,11 +2266,9 @@ out:
                 GF_FREE (xl_opts);
         }
 
-        if (cmd_str)
-                GF_FREE (cmd_str);
+        GF_FREE (cmd_str);
 
-        if (mntpt)
-                GF_FREE (mntpt);
+        GF_FREE (mntpt);
 
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
new file mode 100644
index 000000000..7f094c439
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -0,0 +1,1491 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterd-volgen.h"
+#include "glusterd-utils.h"
+
+static int
+check_dict_key_value (dict_t *dict, char *key, char *value)
+{
+        glusterd_conf_t     *priv          = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Dict.");
+                ret = -1;
+                goto out;
+        }
+
+        if (!key) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Key.");
+                ret = -1;
+                goto out;
+        }
+
+        if (!value) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Value.");
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+get_volname_volinfo (dict_t *dict, char **volname, glusterd_volinfo_t **volinfo)
+{
+        glusterd_conf_t     *priv          = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (*volname, volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_cache_max_min_size (dict_t *dict, char *key, char *value,
+                             char **op_errstr)
+{
+        char                *current_max_value = NULL;
+        char                *current_min_value = NULL;
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        uint64_t             max_value     = 0;
+        uint64_t             min_value     = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        if ((!strcmp (key, "performance.cache-min-file-size")) ||
+            (!strcmp (key, "cache-min-file-size"))) {
+                glusterd_volinfo_get (volinfo,
+                                      "performance.cache-max-file-size",
+                                      &current_max_value);
+                if (current_max_value) {
+                        gf_string2bytesize (current_max_value, &max_value);
+                        gf_string2bytesize (value, &min_value);
+                        current_min_value = value;
+                }
+        } else  if ((!strcmp (key, "performance.cache-max-file-size")) ||
+                    (!strcmp (key, "cache-max-file-size"))) {
+                glusterd_volinfo_get (volinfo,
+                                      "performance.cache-min-file-size",
+                                      &current_min_value);
+                if (current_min_value) {
+                        gf_string2bytesize (current_min_value, &min_value);
+                        gf_string2bytesize (value, &max_value);
+                        current_max_value = value;
+                }
+        }
+
+        if (min_value > max_value) {
+                snprintf (errstr, sizeof (errstr),
+                          "cache-min-file-size (%s) is greater than "
+                          "cache-max-file-size (%s)",
+                          current_min_value, current_max_value);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_quota (dict_t *dict, char *key, char *value,
+                char **op_errstr)
+{
+        char                 errstr[2048] = "";
+        char                *volname      = NULL;
+        glusterd_conf_t     *priv         = NULL;
+        glusterd_volinfo_t  *volinfo      = NULL;
+        int                  ret          = 0;
+        xlator_t            *this         = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get the quota status");
+                goto out;
+        }
+
+        if (ret == _gf_false) {
+                snprintf (errstr, sizeof (errstr),
+                          "Cannot set %s. Enable quota first.", key);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_stripe (dict_t *dict, char *key, char *value, char **op_errstr)
+{
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        if (volinfo->stripe_count == 1) {
+                snprintf (errstr, sizeof (errstr),
+                          "Cannot set %s for a non-stripe volume.", key);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+               goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_subvols_per_directory (dict_t *dict, char *key, char *value,
+                                char **op_errstr)
+{
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        int                  subvols       = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        subvols = atoi(value);
+
+        /* Checking if the subvols-per-directory exceed the total
+           number of subvolumes. */
+        if (subvols > volinfo->subvol_count) {
+                snprintf (errstr, sizeof(errstr),
+                          "subvols-per-directory(%d) is greater "
+                          "than the number of subvolumes(%d).",
+                          subvols, volinfo->subvol_count);
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s.", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+
+/* dispatch table for VOLUME SET
+ * -----------------------------
+ *
+ * Format of entries:
+ *
+ * First field is the <key>, for the purpose of looking it up
+ * in volume dictionary. Each <key> is of the format "<domain>.<specifier>".
+ *
+ * Second field is <voltype>.
+ *
+ * Third field is <option>, if its unset, it's assumed to be
+ * the same as <specifier>.
+ *
+ * Fourth field is <value>. In this context they are used to specify
+ * a default. That is, even the volume dict doesn't have a value,
+ * we procced as if the default value were set for it.
+ *
+ * Fifth field is <doctype>, which decides if the option is public and available
+ * in "set help" or not. "NO_DOC" entries are not part of the public interface
+ * and are subject to change at any time. This also decides if an option is
+ * global (apllies to all volumes) or normal (applies to only specified volume).
+ *
+ * Sixth field is <flags>.
+ *
+ * Seventh field is <op-version>.
+ *
+ * Eight field is description of option: If NULL, tried to fetch from
+ * translator code's xlator_options table.
+ *
+ * Nineth field is validation function: If NULL, xlator's option specific
+ * validation will be tried, otherwise tried at glusterd code itself.
+ *
+ * There are two type of entries: basic and special.
+ *
+ * - Basic entries are the ones where the <option> does _not_ start with
+ *   the bang! character ('!').
+ *
+ *   In their case, <option> is understood as an option for an xlator of
+ *   type <voltype>. Their effect is to copy over the volinfo->dict[<key>]
+ *   value to all graph nodes of type <voltype> (if such a value is set).
+ *
+ *   You are free to add entries of this type, they will become functional
+ *   just by being present in the table.
+ *
+ * - Special entries where the <option> starts with the bang!.
+ *
+ *   They are not applied to all graphs during generation, and you cannot
+ *   extend them in a trivial way which could be just picked up. Better
+ *   not touch them unless you know what you do.
+ *
+ *
+ * Another kind of grouping for options, according to visibility:
+ *
+ * - Exported: one which is used in the code. These are characterized by
+ *   being used a macro as <key> (of the format VKEY_..., defined in
+ *   glusterd-volgen.h
+ *
+ * - Non-exported: the rest; these have string literal <keys>.
+ *
+ * Adhering to this policy, option name changes shall be one-liners.
+ *
+ */
+
+struct volopt_map_entry glusterd_volopt_map[] = {
+        /* DHT xlator options */
+        { .key        = "cluster.lookup-unhashed",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.min-free-disk",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.min-free-inodes",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.rebalance-stats",
+          .voltype    = "cluster/distribute",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "cluster.subvols-per-directory",
+          .voltype     = "cluster/distribute",
+          .option      = "directory-layout-spread",
+          .op_version  = 2,
+          .validate_fn = validate_subvols_per_directory,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.readdir-optimize",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.rsync-hash-regex",
+          .voltype    = "cluster/distribute",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.extra-hash-regex",
+          .voltype    = "cluster/distribute",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.dht-xattr-name",
+          .voltype    = "cluster/distribute",
+          .option     = "xattr-name",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* NUFA xlator options (Distribute special case) */
+        { .key        = "cluster.nufa",
+          .voltype    = "cluster/distribute",
+          .option     = "!nufa",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.local-volume-name",
+          .voltype    = "cluster/nufa",
+          .option     = "local-volume-name",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Switch xlator options (Distribute special case) */
+        { .key        = "cluster.switch",
+          .voltype    = "cluster/distribute",
+          .option     = "!switch",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.switch-pattern",
+          .voltype    = "cluster/switch",
+          .option     = "pattern.switch.case",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* AFR xlator options */
+        { .key        = "cluster.entry-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-subvolume",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags     = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-subvolume-index",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-hash-mode",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.background-self-heal-count",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.metadata-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.entry-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key           = "cluster.self-heal-daemon",
+          .voltype       = "cluster/replicate",
+          .option        = "!self-heal-daemon",
+          .op_version    = 1
+        },
+        { .key        = "cluster.heal-timeout",
+          .voltype    = "cluster/replicate",
+          .option     = "!heal-timeout",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.strict-readdir",
+          .voltype    = "cluster/replicate",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.self-heal-window-size",
+          .voltype    = "cluster/replicate",
+          .option     = "data-self-heal-window-size",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.metadata-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-self-heal-algorithm",
+          .voltype    = "cluster/replicate",
+          .option     = "data-self-heal-algorithm",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.eager-lock",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.quorum-type",
+          .voltype    = "cluster/replicate",
+          .option     = "quorum-type",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.quorum-count",
+          .voltype    = "cluster/replicate",
+          .option     = "quorum-count",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.choose-local",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.self-heal-readdir-size",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.post-op-delay-secs",
+          .voltype    = "cluster/replicate",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.readdir-failover",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.ensure-durability",
+          .voltype    = "cluster/replicate",
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Stripe xlator options */
+        { .key         = "cluster.stripe-block-size",
+          .voltype     = "cluster/stripe",
+          .option      = "block-size",
+          .op_version  = 1,
+          .validate_fn = validate_stripe,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.stripe-coalesce",
+          .voltype    = "cluster/stripe",
+          .option     = "coalesce",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-stats xlator options */
+        { .key         = VKEY_DIAG_LAT_MEASUREMENT,
+          .voltype     = "debug/io-stats",
+          .option      = "latency-measurement",
+          .value       = "off",
+          .op_version  = 1
+        },
+        { .key         = "diagnostics.dump-fd-stats",
+          .voltype     = "debug/io-stats",
+          .op_version  = 1
+        },
+        { .key         = VKEY_DIAG_CNT_FOP_HITS,
+          .voltype     = "debug/io-stats",
+          .option      = "count-fop-hits",
+          .value       = "off",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "diagnostics.brick-log-level",
+          .voltype     = "debug/io-stats",
+          .option      = "!brick-log-level",
+          .op_version  = 1
+        },
+        { .key        = "diagnostics.client-log-level",
+          .voltype    = "debug/io-stats",
+          .option     = "!client-log-level",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "diagnostics.brick-sys-log-level",
+          .voltype     = "debug/io-stats",
+          .option      = "!sys-log-level",
+          .op_version  = 1
+        },
+        { .key        = "diagnostics.client-sys-log-level",
+          .voltype    = "debug/io-stats",
+          .option     = "!sys-log-level",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-cache xlator options */
+        { .key         = "performance.cache-max-file-size",
+          .voltype     = "performance/io-cache",
+          .option      = "max-file-size",
+          .op_version  = 1,
+          .validate_fn = validate_cache_max_min_size,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "performance.cache-min-file-size",
+          .voltype     = "performance/io-cache",
+          .option      = "min-file-size",
+          .op_version  = 1,
+          .validate_fn = validate_cache_max_min_size,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-refresh-timeout",
+          .voltype    = "performance/io-cache",
+          .option     = "cache-timeout",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-priority",
+          .voltype    = "performance/io-cache",
+          .option     = "priority",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-size",
+          .voltype    = "performance/io-cache",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-threads xlator options */
+        { .key         = "performance.io-thread-count",
+          .voltype     = "performance/io-threads",
+          .option      = "thread-count",
+          .op_version  = 1
+        },
+        { .key         = "performance.high-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.normal-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.low-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.least-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.enable-least-priority",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.least-rate-limit",
+          .voltype     = "performance/io-threads",
+          .op_version  = 2
+        },
+
+        /* Other perf xlators' options */
+        { .key        = "performance.cache-size",
+          .voltype    = "performance/quick-read",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.flush-behind",
+          .voltype    = "performance/write-behind",
+          .option     = "flush-behind",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.write-behind-window-size",
+          .voltype    = "performance/write-behind",
+          .option     = "cache-size",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.strict-o-direct",
+          .voltype    = "performance/write-behind",
+          .option     = "strict-O_DIRECT",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.strict-write-ordering",
+          .voltype    = "performance/write-behind",
+          .option     = "strict-write-ordering",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.lazy-open",
+          .voltype    = "performance/open-behind",
+          .option     = "lazy-open",
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.read-after-open",
+          .voltype    = "performance/open-behind",
+          .option     = "read-after-open",
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.read-ahead-page-count",
+          .voltype    = "performance/read-ahead",
+          .option     = "page-count",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.md-cache-timeout",
+          .voltype    = "performance/md-cache",
+          .option     = "md-cache-timeout",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+ 	/* Crypt xlator options */
+
+	{ .key         = "features.encryption",
+	  .voltype     = "encryption/crypt",
+	  .option      = "!feat",
+	  .value       = "off",
+	  .op_version  = 3,
+	  .description = "enable/disable client-side encryption for "
+                         "the volume.",
+	  .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+	},
+
+        { .key         = "encryption.master-key",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "encryption.data-key-size",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "encryption.block-size",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Client xlator options */
+        { .key        = "network.frame-timeout",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.ping-timeout",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.tcp-window-size",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "features.lock-heal",
+          .voltype    = "protocol/client",
+          .option     = "lk-heal",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "features.grace-timeout",
+          .voltype    = "protocol/client",
+          .option     = "grace-timeout",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "client.ssl",
+          .voltype    = "protocol/client",
+          .option     = "transport.socket.ssl-enabled",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.remote-dio",
+          .voltype    = "protocol/client",
+          .option     = "filter-O_DIRECT",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Server xlator options */
+        { .key         = "network.tcp-window-size",
+          .voltype     = "protocol/server",
+          .op_version  = 1
+        },
+        { .key         = "network.inode-lru-limit",
+          .voltype     = "protocol/server",
+          .op_version  = 1
+        },
+        { .key         = AUTH_ALLOW_MAP_KEY,
+          .voltype     = "protocol/server",
+          .option      = "!server-auth",
+          .value       = "*",
+          .op_version  = 1
+        },
+        { .key         = AUTH_REJECT_MAP_KEY,
+          .voltype     = "protocol/server",
+          .option      = "!server-auth",
+          .op_version  = 1
+        },
+        { .key         = "transport.keepalive",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.keepalive",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.allow-insecure",
+          .voltype     = "protocol/server",
+          .option      = "rpc-auth-allow-insecure",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.root-squash",
+          .voltype     = "protocol/server",
+          .option      = "root-squash",
+          .op_version  = 2
+        },
+        { .key         = "server.anonuid",
+          .voltype     = "protocol/server",
+          .option      = "anonuid",
+          .op_version  = 3
+        },
+        { .key         = "server.anongid",
+          .voltype     = "protocol/server",
+          .option      = "anongid",
+          .op_version  = 3
+        },
+        { .key         = "server.statedump-path",
+          .voltype     = "protocol/server",
+          .option      = "statedump-path",
+          .op_version  = 1
+        },
+        { .key         = "server.outstanding-rpc-limit",
+          .voltype     = "protocol/server",
+          .option      = "rpc.outstanding-rpc-limit",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "features.lock-heal",
+          .voltype     = "protocol/server",
+          .option      = "lk-heal",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "features.grace-timeout",
+          .voltype     = "protocol/server",
+          .option      = "grace-timeout",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.ssl",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.ssl-enabled",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+
+        /* Performance xlators enable/disbable options */
+        { .key         = "performance.write-behind",
+          .voltype     = "performance/write-behind",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable write-behind translator in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.read-ahead",
+          .voltype     = "performance/read-ahead",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable read-ahead translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.readdir-ahead",
+          .voltype     = "performance/readdir-ahead",
+          .option      = "!perf",
+          .value       = "off",
+          .op_version  = 3,
+          .description = "enable/disable readdir-ahead translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+
+        { .key         = "performance.io-cache",
+          .voltype     = "performance/io-cache",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable io-cache translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "performance.quick-read",
+          .voltype     = "performance/quick-read",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable quick-read translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+
+        },
+        { .key         = "performance.open-behind",
+          .voltype     = "performance/open-behind",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 2,
+          .description = "enable/disable open-behind translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+
+        },
+        { .key         = "performance.stat-prefetch",
+          .voltype     = "performance/md-cache",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable meta-data caching translator in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.client-io-threads",
+          .voltype     = "performance/io-threads",
+          .option      = "!perf",
+          .value       = "off",
+          .op_version  = 1,
+          .description = "enable/disable io-threads translator in the client "
+                         "graph of volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.write-behind",
+          .voltype    = "performance/write-behind",
+          .option     = "!nfsperf",
+          .value      = "on",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.read-ahead",
+          .voltype    = "performance/read-ahead",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.io-cache",
+          .voltype    = "performance/io-cache",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.quick-read",
+          .voltype    = "performance/quick-read",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.stat-prefetch",
+          .voltype    = "performance/md-cache",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.io-threads",
+          .voltype    = "performance/io-threads",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.force-readdirp",
+          .voltype    = "performance/md-cache",
+          .option     = "force-readdirp",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+	/* Feature translators */
+        { .key         = "features.file-snapshot",
+          .voltype     = "features/qemu-block",
+          .option      = "!feat",
+          .value       = "off",
+          .op_version  = 3,
+          .description = "enable/disable file-snapshot feature in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+
+#ifdef HAVE_LIB_Z
+        /* Compressor-decompressor xlator options
+         * defaults used from xlator/features/compress/src/cdc.h
+         */
+        { .key         = "network.compression",
+          .voltype     = "features/cdc",
+          .option      = "!feat",
+          .value       = "off",
+          .op_version  = 3,
+          .description = "enable/disable network compression translator",
+          .flags       = OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "network.compression.window-size",
+          .voltype     = "features/cdc",
+          .option      = "window-size",
+          .op_version  = 3
+        },
+        { .key         = "network.compression.mem-level",
+          .voltype     = "features/cdc",
+          .option      = "mem-level",
+          .op_version  = 3
+        },
+        { .key         = "network.compression.min-size",
+          .voltype     = "features/cdc",
+          .option      = "min-size",
+          .op_version  = 3
+        },
+        { .key         = "network.compression.compression-level",
+          .voltype     = "features/cdc",
+          .option      = "compression-level",
+          .op_version  = 3
+        },
+        { .key         = "network.compression.debug",
+          .voltype     = "features/cdc",
+          .option      = "debug",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+#endif
+
+        /* Quota xlator options */
+        { .key           = VKEY_FEATURES_LIMIT_USAGE,
+          .voltype       = "features/quota",
+          .option        = "limit-set",
+          .type          = NO_DOC,
+          .op_version    = 1,
+        },
+        {
+          .key           = "features.quota-timeout",
+          .voltype       = "features/quota",
+          .option        = "timeout",
+          .value         = "0",
+          .op_version    = 1,
+          .validate_fn   = validate_quota,
+        },
+        { .key           = "features.default-soft-limit",
+          .voltype       = "features/quota",
+          .option        = "default-soft-limit",
+          .type          = NO_DOC,
+          .op_version    = 3,
+        },
+        { .key           = "features.soft-timeout",
+          .voltype       = "features/quota",
+          .option        = "soft-timeout",
+          .type          = NO_DOC,
+          .op_version    = 3,
+        },
+        { .key           = "features.hard-timeout",
+          .voltype       = "features/quota",
+          .option        = "hard-timeout",
+          .type          = NO_DOC,
+          .op_version    = 3,
+        },
+        { .key           = "features.alert-time",
+          .voltype       = "features/quota",
+          .option        = "alert-time",
+          .type          = NO_DOC,
+          .op_version    = 3,
+        },
+        { .key           = "features.quota-deem-statfs",
+          .voltype       = "features/quota",
+          .option        = "deem-statfs",
+          .value         = "off",
+          .type          = DOC,
+          .op_version    = 2,
+          .validate_fn   = validate_quota,
+        },
+
+        /* Marker xlator options */
+        { .key         = VKEY_MARKER_XTIME,
+          .voltype     = "features/marker",
+          .option      = "xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+        { .key         = VKEY_MARKER_XTIME,
+          .voltype     = "features/marker",
+          .option      = "!xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+        { .key         = VKEY_MARKER_XTIME_FORCE,
+          .voltype     = "features/marker",
+          .option      = "gsync-force-xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 2
+        },
+        { .key         = VKEY_MARKER_XTIME_FORCE,
+          .voltype     = "features/marker",
+          .option      = "!gsync-force-xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 2
+        },
+        { .key         = VKEY_FEATURES_QUOTA,
+          .voltype     = "features/marker",
+          .option      = "quota",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+
+        /* Debug xlators options */
+        { .key        = "debug.trace",
+          .voltype    = "debug/trace",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "debug.log-history",
+          .voltype     = "debug/trace",
+          .option      = "log-history",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.log-file",
+          .voltype     = "debug/trace",
+          .option      = "log-file",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.exclude-ops",
+          .voltype     = "debug/trace",
+          .option      = "exclude-ops",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.include-ops",
+          .voltype     = "debug/trace",
+          .option      = "include-ops",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key        = "debug.error-gen",
+          .voltype    = "debug/error-gen",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "debug.error-failure",
+          .voltype     = "debug/error-gen",
+          .option      = "failure",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.error-number",
+          .voltype     = "debug/error-gen",
+          .option      = "error-no",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.random-failure",
+          .voltype     = "debug/error-gen",
+          .option      = "random-failure",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.error-fops",
+          .voltype     = "debug/error-gen",
+          .option      = "enable",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+
+
+        /* NFS xlator options */
+        { .key         = "nfs.enable-ino32",
+          .voltype     = "nfs/server",
+          .option      = "nfs.enable-ino32",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.mem-factor",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mem-factor",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-dirs",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.export-dirs",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-volumes",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.export-volumes",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.addr-namelookup",
+          .voltype     = "nfs/server",
+          .option      = "rpc-auth.addr.namelookup",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.dynamic-volumes",
+          .voltype     = "nfs/server",
+          .option      = "nfs.dynamic-volumes",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.register-with-portmap",
+          .voltype     = "nfs/server",
+          .option      = "rpc.register-with-portmap",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.outstanding-rpc-limit",
+          .voltype     = "nfs/server",
+          .option      = "rpc.outstanding-rpc-limit",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.port",
+          .voltype     = "nfs/server",
+          .option      = "nfs.port",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-unix",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.auth-unix.*",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-null",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.auth-null.*",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-allow",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.addr.*.allow",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-reject",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.addr.*.reject",
+          .op_version  = 1
+        },
+        { .key         = "nfs.ports-insecure",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.ports.*.insecure",
+          .op_version  = 1
+        },
+        { .key         = "nfs.transport-type",
+          .voltype     = "nfs/server",
+          .option      = "!nfs.transport-type",
+          .value       = "tcp",
+          .op_version  = 1,
+          .description = "Specifies the nfs transport type. Valid "
+                         "transport types are 'tcp' and 'rdma'."
+        },
+        { .key         = "nfs.trusted-sync",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.trusted-sync",
+          .op_version  = 1
+        },
+        { .key         = "nfs.trusted-write",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.trusted-write",
+          .op_version  = 1
+        },
+        { .key         = "nfs.volume-access",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.volume-access",
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-dir",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.export-dir",
+          .op_version  = 1
+        },
+        { .key         = NFS_DISABLE_MAP_KEY,
+          .voltype     = "nfs/server",
+          .option      = "!nfs-disable",
+          .op_version  = 1
+        },
+        { .key         = "nfs.nlm",
+          .voltype     = "nfs/server",
+          .option      = "nfs.nlm",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.acl",
+          .voltype     = "nfs/server",
+          .option      = "nfs.acl",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.mount-udp",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mount-udp",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.mount-rmtab",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mount-rmtab",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.server-aux-gids",
+          .voltype     = "nfs/server",
+          .option      = "nfs.server-aux-gids",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "nfs.drc",
+          .voltype     = "nfs/server",
+          .option      = "nfs.drc",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.drc-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs.drc-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.read-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.read-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.write-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.write-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.readdir-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.readdir-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+
+        /* Other options which don't fit any place above */
+        { .key        = "features.read-only",
+          .voltype    = "features/read-only",
+          .option     = "!read-only",
+          .value      = "off",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "features.worm",
+          .voltype    = "features/worm",
+          .option     = "!worm",
+          .value      = "off",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "storage.linux-aio",
+          .voltype     = "storage/posix",
+          .op_version  = 1
+        },
+        { .key         = "storage.batch-fsync-mode",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.batch-fsync-delay-usec",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.owner-uid",
+          .voltype     = "storage/posix",
+          .option      = "brick-uid",
+          .op_version  = 1
+        },
+        { .key         = "storage.owner-gid",
+          .voltype     = "storage/posix",
+          .option      = "brick-gid",
+          .op_version  = 1
+        },
+        { .key         = "storage.node-uuid-pathinfo",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.health-check-interval",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .option      = "update-link-count-parent",
+          .key         = "storage.build-pgfid",
+          .voltype     = "storage/posix",
+          .op_version  = 4
+        },
+        { .key         = "storage.bd-aio",
+          .voltype     = "storage/bd",
+          .op_version  = 3
+        },
+        { .key        = "config.memory-accounting",
+          .voltype    = "configuration",
+          .option     = "!config",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "config.transport",
+          .voltype     = "configuration",
+          .option      = "!config",
+          .op_version  = 2
+        },
+        { .key         = GLUSTERD_QUORUM_TYPE_KEY,
+          .voltype     = "mgmt/glusterd",
+          .value       = "off",
+          .op_version  = 2
+        },
+        { .key         = GLUSTERD_QUORUM_RATIO_KEY,
+          .voltype     = "mgmt/glusterd",
+          .value       = "0",
+          .op_version  = 2
+        },
+        /* changelog translator - global tunables */
+        { .key         = "changelog.changelog",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "changelog.changelog-dir",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "changelog.encoding",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "changelog.rollover-time",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "changelog.fsync-interval",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = NULL
+        }
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 0f807ca57..a5f694545 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -44,20 +34,27 @@
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-store.h"
+#include "glusterd-hooks.h"
 #include "glusterd-utils.h"
+#include "glusterd-locks.h"
 #include "common-utils.h"
 #include "run.h"
 
+#include "syncop.h"
+
 #include "glusterd-mountbroker.h"
 
-static uuid_t glusterd_uuid;
 extern struct rpcsvc_program gluster_handshake_prog;
+extern struct rpcsvc_program gluster_cli_getspec_prog;
 extern struct rpcsvc_program gluster_pmap_prog;
 extern glusterd_op_info_t opinfo;
 extern struct rpcsvc_program gd_svc_mgmt_prog;
+extern struct rpcsvc_program gd_svc_mgmt_v3_prog;
 extern struct rpcsvc_program gd_svc_peer_prog;
 extern struct rpcsvc_program gd_svc_cli_prog;
+extern struct rpcsvc_program gd_svc_cli_prog_ro;
 extern struct rpc_clnt_program gd_brick_prog;
+extern struct rpcsvc_program glusterd_mgmt_hndsk_prog;
 
 rpcsvc_cbk_program_t glusterd_cbk_prog = {
         .progname  = "Gluster Callback",
@@ -65,6 +62,56 @@ rpcsvc_cbk_program_t glusterd_cbk_prog = {
         .progver   = GLUSTER_CBK_VERSION,
 };
 
+struct rpcsvc_program *gd_inet_programs[] = {
+        &gd_svc_peer_prog,
+        &gd_svc_cli_prog_ro,
+        &gd_svc_mgmt_prog,
+        &gd_svc_mgmt_v3_prog,
+        &gluster_pmap_prog,
+        &gluster_handshake_prog,
+        &glusterd_mgmt_hndsk_prog,
+};
+int gd_inet_programs_count = (sizeof (gd_inet_programs) /
+                              sizeof (gd_inet_programs[0]));
+
+struct rpcsvc_program *gd_uds_programs[] = {
+        &gd_svc_cli_prog,
+        &gluster_cli_getspec_prog,
+};
+int gd_uds_programs_count = (sizeof (gd_uds_programs) /
+                             sizeof (gd_uds_programs[0]));
+
+const char *gd_op_list[GD_OP_MAX + 1] = {
+        [GD_OP_NONE]                    = "Invalid op",
+        [GD_OP_CREATE_VOLUME]           = "Create",
+        [GD_OP_START_BRICK]             = "Start Brick",
+        [GD_OP_STOP_BRICK]              = "Stop Brick",
+        [GD_OP_DELETE_VOLUME]           = "Delete",
+        [GD_OP_START_VOLUME]            = "Start",
+        [GD_OP_STOP_VOLUME]             = "Stop",
+        [GD_OP_DEFRAG_VOLUME]           = "Rebalance",
+        [GD_OP_ADD_BRICK]               = "Add brick",
+        [GD_OP_REMOVE_BRICK]            = "Remove brick",
+        [GD_OP_REPLACE_BRICK]           = "Replace brick",
+        [GD_OP_SET_VOLUME]              = "Set",
+        [GD_OP_RESET_VOLUME]            = "Reset",
+        [GD_OP_SYNC_VOLUME]             = "Sync",
+        [GD_OP_LOG_ROTATE]              = "Log rotate",
+        [GD_OP_GSYNC_SET]               = "Geo-replication",
+        [GD_OP_PROFILE_VOLUME]          = "Profile",
+        [GD_OP_QUOTA]                   = "Quota",
+        [GD_OP_STATUS_VOLUME]           = "Status",
+        [GD_OP_REBALANCE]               = "Rebalance",
+        [GD_OP_HEAL_VOLUME]             = "Heal",
+        [GD_OP_STATEDUMP_VOLUME]        = "Statedump",
+        [GD_OP_LIST_VOLUME]             = "Lists",
+        [GD_OP_CLEARLOCKS_VOLUME]       = "Clear locks",
+        [GD_OP_DEFRAG_BRICK_VOLUME]     = "Rebalance",
+        [GD_OP_COPY_FILE]               = "Copy File",
+        [GD_OP_SYS_EXEC]                = "Execute system commands",
+        [GD_OP_GSYNC_CREATE]            = "Geo-replication Create",
+        [GD_OP_MAX]                     = "Invalid op"
+};
 
 static int
 glusterd_opinfo_init ()
@@ -76,35 +123,30 @@ glusterd_opinfo_init ()
         return ret;
 }
 
-static int
-glusterd_uuid_init (int flag)
+
+int
+glusterd_uuid_init ()
 {
         int             ret = -1;
+        xlator_t        *this = NULL;
         glusterd_conf_t *priv = NULL;
 
-        priv = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
 
-        if (!flag) {
-                ret = glusterd_retrieve_uuid ();
-                if (!ret) {
-                        uuid_copy (glusterd_uuid, priv->uuid);
-                        gf_log ("glusterd", GF_LOG_INFO,
-                                "retrieved UUID: %s", uuid_utoa (priv->uuid));
-                        return 0;
-                }
+        ret = glusterd_retrieve_uuid ();
+        if (ret == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "retrieved UUID: %s", uuid_utoa (priv->uuid));
+                return 0;
         }
 
-        uuid_generate (glusterd_uuid);
-
-        gf_log ("glusterd", GF_LOG_INFO,
-                        "generated UUID: %s", uuid_utoa (glusterd_uuid));
-        uuid_copy (priv->uuid, glusterd_uuid);
-
-        ret = glusterd_store_uuid ();
+        ret = glusterd_uuid_generate_save ();
 
         if (ret) {
                 gf_log ("glusterd", GF_LOG_ERROR,
-                          "Unable to store generated UUID");
+                          "Unable to generate and save new UUID");
                 return ret;
         }
 
@@ -112,6 +154,63 @@ glusterd_uuid_init (int flag)
 }
 
 int
+glusterd_uuid_generate_save ()
+{
+        int               ret = -1;
+        glusterd_conf_t   *priv = NULL;
+        xlator_t          *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        uuid_generate (priv->uuid);
+
+        gf_log (this->name, GF_LOG_INFO, "generated UUID: %s",
+                uuid_utoa (priv->uuid));
+
+        ret = glusterd_store_global_info (this);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to store the generated uuid %s",
+                        uuid_utoa (priv->uuid));
+
+        return ret;
+}
+
+int
+glusterd_options_init (xlator_t *this)
+{
+        int             ret = -1;
+        glusterd_conf_t *priv = NULL;
+        char            *initial_version = "0";
+
+        priv = this->private;
+
+        priv->opts = dict_new ();
+        if (!priv->opts)
+                goto out;
+
+        ret = glusterd_store_retrieve_options (this);
+        if (ret == 0)
+                goto out;
+
+        ret = dict_set_str (priv->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                            initial_version);
+        if (ret)
+                goto out;
+        ret = glusterd_store_options (this, priv->opts);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to store version");
+                return ret;
+        }
+out:
+
+        return 0;
+}
+int
 glusterd_fetchspec_notify (xlator_t *this)
 {
         int              ret   = -1;
@@ -120,10 +219,15 @@ glusterd_fetchspec_notify (xlator_t *this)
 
         priv = this->private;
 
-        list_for_each_entry (trans, &priv->xprt_list, list) {
-                rpcsvc_callback_submit (priv->rpc, trans, &glusterd_cbk_prog,
-                                        GF_CBK_FETCHSPEC, NULL, 0);
+        pthread_mutex_lock (&priv->xprt_lock);
+        {
+                list_for_each_entry (trans, &priv->xprt_list, list) {
+                        rpcsvc_callback_submit (priv->rpc, trans,
+                                                &glusterd_cbk_prog,
+                                                GF_CBK_FETCHSPEC, NULL, 0);
+                }
         }
+        pthread_mutex_unlock (&priv->xprt_lock);
 
         ret = 0;
 
@@ -181,12 +285,16 @@ glusterd_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
         {
                 INIT_LIST_HEAD (&xprt->list);
 
+                pthread_mutex_lock (&priv->xprt_lock);
                 list_add_tail (&xprt->list, &priv->xprt_list);
+                pthread_mutex_unlock (&priv->xprt_lock);
                 break;
         }
         case RPCSVC_EVENT_DISCONNECT:
         {
+                pthread_mutex_lock (&priv->xprt_lock);
                 list_del (&xprt->list);
+                pthread_mutex_unlock (&priv->xprt_lock);
                 pmap_registry_remove (this, 0, NULL, GF_PMAP_PORT_NONE, xprt);
                 break;
         }
@@ -243,18 +351,6 @@ out:
         return ret;
 }
 
-/* defined in glusterd-utils.c -- no
- * glusterd header where it would be
- * appropriate to put to, and too
- * accidental routine to place in
- * libglusterfs.
- *
- * (Indeed, XXX: we'd rather need a general
- * "mkdir -p" like routine in
- * libglusterfs)
- */
-extern int mkdir_if_missing (char *path);
-
 #if SYNCDAEMON_COMPILE
 static int
 glusterd_check_gsync_present (int *valid_state)
@@ -347,13 +443,13 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
         if (strlen (conf->workdir)+2 > PATH_MAX-strlen(GEOREP)) {
                 ret = -1;
                 gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" directory %s",
-                        georepdir);
+                        "directory path %s/"GEOREP" is longer than PATH_MAX",
+                        conf->workdir);
                 goto out;
         }
 
         snprintf (georepdir, PATH_MAX, "%s/"GEOREP, conf->workdir);
-        ret = mkdir_if_missing (georepdir);
+        ret = mkdir_p (georepdir, 0777, _gf_true);
         if (-1 == ret) {
                 gf_log ("glusterd", GF_LOG_CRITICAL,
                         "Unable to create "GEOREP" directory %s",
@@ -364,31 +460,49 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
         if (strlen (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP) >= PATH_MAX) {
                 ret = -1;
                 gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" directory %s",
-                        georepdir);
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
+                        GEOREP" is longer than PATH_MAX");
                 goto out;
         }
-        ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP);
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP, 0777, _gf_true);
         if (-1 == ret) {
                 gf_log ("glusterd", GF_LOG_CRITICAL,
                         "Unable to create "GEOREP" log directory");
                 goto out;
         }
 
+        /* Slave log file directory */
         if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves") >= PATH_MAX) {
                 ret = -1;
                 gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" directory %s",
-                        georepdir);
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
+                        GEOREP"-slaves"" is longer than PATH_MAX");
                 goto out;
         }
-        ret = mkdir_if_missing (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves");
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves", 0777,
+                      _gf_true);
         if (-1 == ret) {
                 gf_log ("glusterd", GF_LOG_CRITICAL,
                         "Unable to create "GEOREP" slave log directory");
                 goto out;
         }
 
+        /* MountBroker log file directory */
+        if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr") >= PATH_MAX) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP
+                        "-slaves/mbr"" is longer than PATH_MAX");
+                goto out;
+        }
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr", 0777,
+                      _gf_true);
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" mountbroker slave log directory");
+                goto out;
+        }
+
         ret = dict_get_str (THIS->options, GEOREP"-log-group", &greplg_s);
         if (ret)
                 ret = 0;
@@ -406,20 +520,22 @@ glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
                 if (ret == 0)
                         ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
                                                  GEOREP"-slaves", gr->gr_gid);
+                if (ret == 0)
+                        ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
+                                                 GEOREP"-slaves/mbr", gr->gr_gid);
         }
 
  out:
         gf_log("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
-#endif
 
 static void
 runinit_gsyncd_setrx (runner_t *runner, glusterd_conf_t *conf)
 {
         runinit (runner);
         runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (runner, "%s/"GSYNC_CONF,conf->workdir);
+        runner_argprintf (runner, "%s/"GSYNC_CONF_TEMPLATE, conf->workdir);
         runner_add_arg (runner, "--config-set-rx");
 }
 
@@ -436,7 +552,6 @@ configure_syncdaemon (glusterd_conf_t *conf)
 } while (0)
 {
         int ret = 0;
-#if SYNCDAEMON_COMPILE
         runner_t runner = {0,};
         char georepdir[PATH_MAX] = {0,};
         int valid_state = 0;
@@ -469,8 +584,8 @@ configure_syncdaemon (glusterd_conf_t *conf)
         RUN_GSYNCD_CMD;
 
         runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "remote-gsyncd",
-                         "/usr/local/libexec/glusterfs/gsyncd", ".", "^ssh:", NULL);
+        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
+                         ".", "^ssh:", NULL);
         RUN_GSYNCD_CMD;
 
         /* gluster-command-dir */
@@ -482,7 +597,7 @@ configure_syncdaemon (glusterd_conf_t *conf)
         /* gluster-params */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner, "gluster-params",
-                         "xlator-option=*-dht.assert-no-child-down=true",
+                         "aux-gfid-mount",
                          ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
@@ -496,20 +611,65 @@ configure_syncdaemon (glusterd_conf_t *conf)
         runner_add_args (&runner, ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
+        /* ssh-command tar */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "ssh-command-tar");
+        runner_argprintf (&runner,
+                          "ssh -oPasswordAuthentication=no "
+                           "-oStrictHostKeyChecking=no "
+                           "-i %s/tar_ssh.pem", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
         /* pid-file */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_arg (&runner, "pid-file");
-        runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.pid", georepdir);
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* geo-rep working dir */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "georep-session-working-dir");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/", georepdir);
         runner_add_args (&runner, ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
         /* state-file */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_arg (&runner, "state-file");
-        runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.status", georepdir);
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+                          georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+                          georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-socket */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-socket-unencoded");
+        runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.socket", georepdir);
         runner_add_args (&runner, ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
+        /* socketdir */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
         /* log-file */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner,
@@ -522,10 +682,32 @@ configure_syncdaemon (glusterd_conf_t *conf)
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner,
                          "gluster-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.gluster.log",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
                          ".", ".", NULL);
         RUN_GSYNCD_CMD;
 
+        /* ignore-deletes */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* special-sync-mode */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* change-detector == changelog */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg(&runner, "working-dir");
+        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                         DEFAULT_VAR_RUN_DIRECTORY);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
         /************
          * slave pre-configuration
          ************/
@@ -539,7 +721,7 @@ configure_syncdaemon (glusterd_conf_t *conf)
         /* gluster-params */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner, "gluster-params",
-                         "xlator-option=*-dht.assert-no-child-down=true",
+                         "aux-gfid-mount",
                          ".", NULL);
         RUN_GSYNCD_CMD;
 
@@ -551,6 +733,14 @@ configure_syncdaemon (glusterd_conf_t *conf)
                          ".", NULL);
         RUN_GSYNCD_CMD;
 
+        /* MountBroker log-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner,
+                         "log-file-mbr",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
         /* gluster-log-file */
         runinit_gsyncd_setrx (&runner, conf);
         runner_add_args (&runner,
@@ -560,12 +750,17 @@ configure_syncdaemon (glusterd_conf_t *conf)
         RUN_GSYNCD_CMD;
 
  out:
-#else
-        (void)conf;
-#endif
         return ret ? -1 : 0;
 }
 #undef RUN_GSYNCD_CMD
+#else /* SYNCDAEMON_COMPILE */
+static int
+configure_syncdaemon (glusterd_conf_t *conf)
+{
+        return 0;
+}
+#endif /* !SYNCDAEMON_COMPILE */
+
 
 static int
 check_prepare_mountbroker_root (char *mountbroker_root)
@@ -597,6 +792,11 @@ check_prepare_mountbroker_root (char *mountbroker_root)
                 ret = -1;
                 goto out;
         }
+        if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
+                gf_log ("", GF_LOG_WARNING,
+                        "permissions on mountbroker-root directory %s are "
+                        "probably too strict", mountbroker_root);
+        }
 
         dfd0 = dup (dfd);
 
@@ -625,6 +825,11 @@ check_prepare_mountbroker_root (char *mountbroker_root)
                         ret = -1;
                         goto out;
                 }
+                if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
+                        gf_log ("", GF_LOG_WARNING,
+                                "permissions on ancestors of mountbroker-root "
+                                "directory are probably too strict");
+                }
 
                 close (dfd);
                 dfd = dfd2;
@@ -647,14 +852,17 @@ check_prepare_mountbroker_root (char *mountbroker_root)
         ret = 0;
 
  out:
-        close (dfd0);
-        close (dfd);
-        close (dfd2);
+        if (dfd0 != -1)
+                close (dfd0);
+        if (dfd != -1)
+                close (dfd);
+        if (dfd2 != -1)
+                close (dfd2);
 
         return ret;
 }
 
-static void
+static int
 _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
 {
         glusterd_conf_t *priv           = THIS->private;
@@ -663,15 +871,11 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
         gf_boolean_t     ghadoop        = _gf_false;
         char            *pdesc          = value->data;
         char            *volname        = NULL;
-        int             *ret            = data;
         int              rv             = 0;
         gf_mount_spec_t *mspec          = NULL;
         char            *user           = NULL;
         char            *volfile_server = NULL;
 
-        if (*ret == -1)
-                return;
-
         label = strtail (key, "mountbroker.");
 
         /* check for presence of geo-rep/hadoop label */
@@ -687,7 +891,7 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
         }
 
         if (!label)
-                return;
+                return 0;
 
         mspec = GF_CALLOC (1, sizeof (*mspec), gf_gld_mt_mount_spec);
         if (!mspec)
@@ -726,14 +930,166 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
 
         list_add_tail (&mspec->speclist, &priv->mount_specs);
 
-        return;
+        return 0;
  err:
 
         gf_log ("", GF_LOG_ERROR,
                 "adding %smount spec failed: label: %s desc: %s",
                 georep ? GEOREP" " : "", label, pdesc);
 
-        *ret = -1;
+        if (mspec) {
+                if (mspec->patterns) {
+                        GF_FREE (mspec->patterns->components);
+                        GF_FREE (mspec->patterns);
+                }
+                GF_FREE (mspec);
+        }
+
+        return -1;
+}
+
+
+static int
+gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+        glusterd_conf_t     *priv = THIS->private;
+        synclock_unlock (&priv->big_lock);
+        return ret;
+}
+
+static void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
+{
+        xlator_t        *this = NULL;
+        glusterd_conf_t *priv = NULL;
+        int             ret   = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        synclock_lock (&priv->big_lock);
+        ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
+                            opaque);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
+                        " and other volume related services");
+}
+
+int
+glusterd_uds_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                            void *data)
+{
+        /* glusterd_rpcsvc_notify() does stuff that calls coming in from the
+         * unix domain socket don't need. This is just an empty function to be
+         * used for the uds listener. This will be used later if required.
+         */
+        return 0;
+}
+
+/* The glusterd unix domain socket listener only listens for cli */
+rpcsvc_t *
+glusterd_init_uds_listener (xlator_t *this)
+{
+        int             ret = -1;
+        dict_t          *options = NULL;
+        rpcsvc_t        *rpc = NULL;
+        data_t          *sock_data = NULL;
+        char            sockfile[PATH_MAX+1] = {0,};
+        int             i = 0;
+
+
+        GF_ASSERT (this);
+
+        sock_data = dict_get (this->options, "glusterd-sockfile");
+        if (!sock_data) {
+                strncpy (sockfile, DEFAULT_GLUSTERD_SOCKFILE, PATH_MAX);
+        } else {
+                strncpy (sockfile, sock_data->data, PATH_MAX);
+        }
+
+        options = dict_new ();
+        if (!options)
+                goto out;
+
+        ret = rpcsvc_transport_unix_options_build (&options, sockfile);
+        if (ret)
+                goto out;
+
+        rpc = rpcsvc_init (this, this->ctx, options, 8);
+        if (rpc == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = rpcsvc_register_notify (rpc, glusterd_uds_rpcsvc_notify,
+                                      this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Failed to register notify function");
+                goto out;
+        }
+
+        ret = rpcsvc_create_listeners (rpc, options, this->name);
+        if (ret != 1) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to create listener");
+                goto out;
+        }
+        ret = 0;
+
+        for (i = 0; i < gd_uds_programs_count; i++) {
+                ret = glusterd_program_register (this, rpc, gd_uds_programs[i]);
+                if (ret) {
+                        i--;
+                        for (; i >= 0; i--)
+                                rpcsvc_program_unregister (rpc,
+                                                           gd_uds_programs[i]);
+
+                        goto out;
+                }
+        }
+
+out:
+        if (options)
+                dict_unref (options);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to start glusterd "
+                        "unix domain socket listener.");
+                if (rpc) {
+                        GF_FREE (rpc);
+                        rpc = NULL;
+                }
+        }
+        return rpc;
+}
+
+void
+glusterd_stop_uds_listener (xlator_t *this)
+{
+        glusterd_conf_t         *conf = NULL;
+        rpcsvc_listener_t       *listener = NULL;
+        rpcsvc_listener_t       *next = NULL;
+
+        GF_ASSERT (this);
+        conf = this->private;
+
+        (void) rpcsvc_program_unregister (conf->uds_rpc, &gd_svc_cli_prog);
+        (void) rpcsvc_program_unregister (conf->uds_rpc, &gluster_handshake_prog);
+
+        list_for_each_entry_safe (listener, next, &conf->uds_rpc->listeners,
+                                  list) {
+                rpcsvc_listener_destroy (listener);
+        }
+
+        (void) rpcsvc_unregister_notify (conf->uds_rpc, glusterd_rpcsvc_notify,
+                                         this);
+
+        unlink (DEFAULT_GLUSTERD_SOCKFILE);
+
+        GF_FREE (conf->uds_rpc);
+        conf->uds_rpc = NULL;
+
+        return;
 }
 
 /*
@@ -747,57 +1103,60 @@ init (xlator_t *this)
 {
         int32_t            ret               = -1;
         rpcsvc_t          *rpc               = NULL;
+        rpcsvc_t          *uds_rpc           = NULL;
         glusterd_conf_t   *conf              = NULL;
         data_t            *dir_data          = NULL;
         struct stat        buf               = {0,};
-        char               voldir [PATH_MAX] = {0,};
-        char               dirname [PATH_MAX];
+        char               storedir [PATH_MAX] = {0,};
+        char               workdir [PATH_MAX] = {0,};
+        char               hooks_dir [PATH_MAX] = {0,};
         char               cmd_log_filename [PATH_MAX] = {0,};
         int                first_time        = 0;
         char              *mountbroker_root  = NULL;
-
-#ifdef DEBUG
+        int                i                 = 0;
         char              *valgrind_str      = NULL;
-#endif
+
         dir_data = dict_get (this->options, "working-directory");
 
         if (!dir_data) {
                 //Use default working dir
-                strncpy (dirname, GLUSTERD_DEFAULT_WORKDIR, PATH_MAX);
+                strncpy (workdir, GLUSTERD_DEFAULT_WORKDIR, PATH_MAX);
         } else {
-                strncpy (dirname, dir_data->data, PATH_MAX);
+                strncpy (workdir, dir_data->data, PATH_MAX);
         }
 
-        ret = stat (dirname, &buf);
+        ret = stat (workdir, &buf);
         if ((ret != 0) && (ENOENT != errno)) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "stat fails on %s, exiting. (errno = %d)",
-			dirname, errno);
+                        workdir, errno);
                 exit (1);
         }
 
         if ((!ret) && (!S_ISDIR(buf.st_mode))) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Provided working area %s is not a directory,"
-                        "exiting", dirname);
+                        "exiting", workdir);
                 exit (1);
         }
 
 
         if ((-1 == ret) && (ENOENT == errno)) {
-                ret = mkdir (dirname, 0777);
+                ret = mkdir (workdir, 0777);
 
                 if (-1 == ret) {
                         gf_log (this->name, GF_LOG_CRITICAL,
                                 "Unable to create directory %s"
-                                " ,errno = %d", dirname, errno);
+                                " ,errno = %d", workdir, errno);
                         exit (1);
                 }
+
                 first_time = 1;
         }
 
+        setenv ("GLUSTERD_WORKING_DIR", workdir, 1);
         gf_log (this->name, GF_LOG_INFO, "Using %s as working directory",
-                dirname);
+                workdir);
 
         snprintf (cmd_log_filename, PATH_MAX,"%s/.cmd_log_history",
                   DEFAULT_LOG_FILE_DIRECTORY);
@@ -809,52 +1168,70 @@ init (xlator_t *this)
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/vols", dirname);
+        snprintf (storedir, PATH_MAX, "%s/vols", workdir);
 
-        ret = mkdir (voldir, 0777);
+        ret = mkdir (storedir, 0777);
 
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create volume directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/peers", dirname);
+        snprintf (storedir, PATH_MAX, "%s/peers", workdir);
 
-        ret = mkdir (voldir, 0777);
+        ret = mkdir (storedir, 0777);
 
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create peers directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/bricks", DEFAULT_LOG_FILE_DIRECTORY);
-        ret = mkdir (voldir, 0777);
+        snprintf (storedir, PATH_MAX, "%s/bricks", DEFAULT_LOG_FILE_DIRECTORY);
+        ret = mkdir (storedir, 0777);
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create logs directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/nfs", dirname);
-        ret = mkdir (voldir, 0777);
+        snprintf (storedir, PATH_MAX, "%s/nfs", workdir);
+        ret = mkdir (storedir, 0777);
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create nfs directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/glustershd", workdir);
+        ret = mkdir (storedir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create glustershd directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/quotad", workdir);
+        ret = mkdir (storedir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create quotad directory %s"
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
-        snprintf (voldir, PATH_MAX, "%s/glustershd", dirname);
-        ret = mkdir (voldir, 0777);
+        snprintf (storedir, PATH_MAX, "%s/groups", workdir);
+        ret = mkdir (storedir, 0777);
         if ((-1 == ret) && (errno != EEXIST)) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "Unable to create glustershd directory %s"
-                        " ,errno = %d", voldir, errno);
+                        " ,errno = %d", storedir, errno);
                 exit (1);
         }
 
@@ -887,55 +1264,51 @@ init (xlator_t *this)
                 goto out;
         }
 
-        ret = glusterd_program_register (this, rpc, &gd_svc_peer_prog);
-        if (ret) {
-                goto out;
-        }
+        for (i = 0; i < gd_inet_programs_count; i++) {
+                ret = glusterd_program_register (this, rpc,
+                                                 gd_inet_programs[i]);
+                if (ret) {
+                        i--;
+                        for (; i >= 0; i--)
+                                rpcsvc_program_unregister (rpc,
+                                                           gd_inet_programs[i]);
 
-        ret = glusterd_program_register (this, rpc, &gd_svc_cli_prog);
-        if (ret) {
-                rpcsvc_program_unregister (rpc, &gd_svc_peer_prog);
-                goto out;
-        }
-
-        ret = glusterd_program_register (this, rpc, &gd_svc_mgmt_prog);
-        if (ret) {
-                rpcsvc_program_unregister (rpc, &gd_svc_peer_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
-                goto out;
-        }
-
-        ret = glusterd_program_register (this, rpc, &gluster_pmap_prog);
-        if (ret) {
-                rpcsvc_program_unregister (rpc, &gd_svc_peer_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_mgmt_prog);
-                goto out;
+                        goto out;
+                }
         }
 
-        ret = glusterd_program_register (this, rpc, &gluster_handshake_prog);
-        if (ret) {
-                rpcsvc_program_unregister (rpc, &gd_svc_peer_prog);
-                rpcsvc_program_unregister (rpc, &gluster_pmap_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_cli_prog);
-                rpcsvc_program_unregister (rpc, &gd_svc_mgmt_prog);
+        /* Start a unix domain socket listener just for cli commands
+         * This should prevent ports from being wasted by being in TIMED_WAIT
+         * when cli commands are done continuously
+         */
+        uds_rpc = glusterd_init_uds_listener (this);
+        if (uds_rpc == NULL) {
+                ret = -1;
                 goto out;
         }
 
         conf = GF_CALLOC (1, sizeof (glusterd_conf_t),
                           gf_gld_mt_glusterd_conf_t);
         GF_VALIDATE_OR_GOTO(this->name, conf, out);
-        conf->shd = GF_CALLOC (1, sizeof (nodesrv_t),
-                               gf_gld_mt_nodesrv_t);
+
+        conf->shd = GF_CALLOC (1, sizeof (nodesrv_t), gf_gld_mt_nodesrv_t);
         GF_VALIDATE_OR_GOTO(this->name, conf->shd, out);
+        conf->nfs = GF_CALLOC (1, sizeof (nodesrv_t), gf_gld_mt_nodesrv_t);
+        GF_VALIDATE_OR_GOTO(this->name, conf->nfs, out);
+        conf->quotad = GF_CALLOC (1, sizeof (nodesrv_t),
+                               gf_gld_mt_nodesrv_t);
+        GF_VALIDATE_OR_GOTO(this->name, conf->quotad, out);
 
         INIT_LIST_HEAD (&conf->peers);
         INIT_LIST_HEAD (&conf->volumes);
         pthread_mutex_init (&conf->mutex, NULL);
         conf->rpc = rpc;
+        conf->uds_rpc = uds_rpc;
         conf->gfs_mgmt = &gd_brick_prog;
-        strncpy (conf->workdir, dirname, PATH_MAX);
+        strncpy (conf->workdir, workdir, PATH_MAX);
 
+        synclock_init (&conf->big_lock);
+        pthread_mutex_init (&conf->xprt_lock, NULL);
         INIT_LIST_HEAD (&conf->xprt_list);
 
         glusterd_friend_sm_init ();
@@ -948,31 +1321,44 @@ init (xlator_t *this)
         if (ret)
                 goto out;
 
+         conf->base_port = GF_IANA_PRIV_PORTS_START;
+         if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
+                 gf_log (this->name, GF_LOG_INFO,
+                         "base-port override: %d", conf->base_port);
+         }
+
         /* Set option to run bricks on valgrind if enabled in glusterd.vol */
-#ifdef DEBUG
         conf->valgrind = _gf_false;
-        ret = dict_get_str (this->options, "brick-with-valgrind", &valgrind_str);
+        ret = dict_get_str (this->options, "run-with-valgrind", &valgrind_str);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "cannot get brick-with-valgrind value");
+                        "cannot get run-with-valgrind value");
         }
         if (valgrind_str) {
                 if (gf_string2boolean (valgrind_str, &(conf->valgrind))) {
                         gf_log (this->name, GF_LOG_WARNING,
-                                "brick-with-valgrind value not a boolean string");
+                                "run-with-valgrind value not a boolean string");
                 }
         }
-#endif
+
         this->private = conf;
-        (void) glusterd_shd_set_running (_gf_false);
-        /* this->ctx->top = this;*/
+        glusterd_vol_lock_init ();
+        glusterd_txn_opinfo_dict_init ();
+        (void) glusterd_nodesvc_set_online_status ("glustershd", _gf_false);
 
-        ret = glusterd_uuid_init (first_time);
-        if (ret < 0)
-                goto out;
+        GLUSTERD_GET_HOOKS_DIR (hooks_dir, GLUSTERD_HOOK_VER, conf);
+        if (stat (hooks_dir, &buf)) {
+                ret = glusterd_hooks_create_hooks_directory (conf->workdir);
+                if (-1 == ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to create hooks directory ");
+                        exit (1);
+                }
+        }
 
         INIT_LIST_HEAD (&conf->mount_specs);
-        dict_foreach (this->options, _install_mount_spec, &ret);
+
+        ret = dict_foreach (this->options, _install_mount_spec, NULL);
         if (ret)
                 goto out;
         ret = dict_get_str (this->options, "mountbroker-root",
@@ -992,16 +1378,25 @@ init (xlator_t *this)
         if (ret < 0)
                 goto out;
 
+        /* If there are no 'friends', this would be the best time to
+         * spawn process/bricks that may need (re)starting since last
+         * time (this) glusterd was up.*/
+
+        if (list_empty (&conf->peers)) {
+                glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
+        }
+        ret = glusterd_options_init (this);
+        if (ret < 0)
+                goto out;
+
         ret = glusterd_handle_upgrade_downgrade (this->options, conf);
         if (ret)
                 goto out;
 
-        glusterd_restart_bricks (conf);
-        ret = glusterd_restart_gsyncds (conf);
+        ret = glusterd_hooks_spawn_worker (this);
         if (ret)
                 goto out;
 
-        glusterd_restart_rebalance (conf);
         ret = 0;
 out:
         if (ret < 0) {
@@ -1033,12 +1428,17 @@ fini (xlator_t *this)
                 goto out;
 
         conf = this->private;
-        if (conf->pmap)
-                FREE (conf->pmap);
+
+        glusterd_stop_uds_listener (this);
+
+        FREE (conf->pmap);
         if (conf->handle)
-                glusterd_store_handle_destroy (conf->handle);
+                gf_store_handle_destroy (conf->handle);
         glusterd_sm_tr_log_delete (&conf->op_sm_log);
+        glusterd_vol_lock_fini ();
+        glusterd_txn_opinfo_dict_fini ();
         GF_FREE (conf);
+
         this->private = NULL;
 out:
         return;
@@ -1076,11 +1476,9 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 }
 
 
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct xlator_dumpops dumpops = {
         .priv  = glusterd_priv,
@@ -1127,10 +1525,33 @@ struct volume_options options[] = {
         { .key = {GEOREP"-log-group"},
           .type = GF_OPTION_TYPE_ANY,
         },
-#ifdef DEBUG
-        { .key = {"brick-with-valgrind"},
+        { .key = {"run-with-valgrind"},
           .type = GF_OPTION_TYPE_BOOL,
         },
-#endif
+        { .key = {"server-quorum-type"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = { "none", "server"},
+          .description = "This feature is on the server-side i.e. in glusterd."
+                         " Whenever the glusterd on a machine observes that "
+                         "the quorum is not met, it brings down the bricks to "
+                         "prevent data split-brains. When the network "
+                         "connections are brought back up and the quorum is "
+                         "restored the bricks in the volume are brought back "
+                         "up."
+        },
+        { .key = {"server-quorum-ratio"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .description = "Sets the quorum percentage for the trusted "
+          "storage pool."
+        },
+        { .key = {"glusterd-sockfile"},
+          .type = GF_OPTION_TYPE_PATH,
+          .description = "The socket file on which glusterd should listen for "
+                        "cli requests. Default is "DEFAULT_GLUSTERD_SOCKFILE "."
+        },
+        { .key = {"base-port"},
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Sets the base port for portmap query"
+        },
         { .key   = {NULL} },
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index e8193bba2..0694f7386 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _GLUSTERD_H_
 #define _GLUSTERD_H_
 
@@ -46,13 +36,41 @@
 #include "protocol-common.h"
 #include "glusterd-pmap.h"
 #include "cli1-xdr.h"
+#include "syncop.h"
+#include "store.h"
 
 #define GLUSTERD_MAX_VOLUME_NAME        1000
-#define DEFAULT_LOG_FILE_DIRECTORY      DATADIR "/log/glusterfs"
 #define GLUSTERD_TR_LOG_SIZE            50
 #define GLUSTERD_NAME                   "glusterd"
 #define GLUSTERD_SOCKET_LISTEN_BACKLOG  128
+#define GLUSTERD_QUORUM_TYPE_KEY        "cluster.server-quorum-type"
+#define GLUSTERD_QUORUM_RATIO_KEY       "cluster.server-quorum-ratio"
+#define GLUSTERD_GLOBAL_OPT_VERSION     "global-option-version"
+#define GLUSTERD_COMMON_PEM_PUB_FILE    "/geo-replication/common_secret.pem.pub"
+#define GEO_CONF_MAX_OPT_VALS           6
+#define GLUSTERD_CREATE_HOOK_SCRIPT     "/hooks/1/gsync-create/post/" \
+                                        "S56glusterd-geo-rep-create-post.sh"
+
+
+#define GLUSTERD_SERVER_QUORUM "server"
+
+#define FMTSTR_CHECK_VOL_EXISTS "Volume %s does not exist"
+#define FMTSTR_RESOLVE_BRICK "Could not find peer on which brick %s:%s resides"
 
+#define LOGSTR_FOUND_BRICK  "Found brick %s:%s in volume %s"
+#define LOGSTR_BUILD_PAYLOAD "Failed to build payload for operation 'Volume %s'"
+#define LOGSTR_STAGE_FAIL "Staging of operation 'Volume %s' failed on %s %s %s"
+#define LOGSTR_COMMIT_FAIL "Commit of operation 'Volume %s' failed on %s %s %s"
+
+#define OPERRSTR_BUILD_PAYLOAD "Failed to build payload. Please check the log "\
+                               "file for more details."
+#define OPERRSTR_STAGE_FAIL "Staging failed on %s. Please check the log file " \
+                            "for more details."
+#define OPERRSTR_COMMIT_FAIL "Commit failed on %s. Please check the log file "\
+                             "for more details."
+
+struct glusterd_volinfo_;
+typedef struct glusterd_volinfo_ glusterd_volinfo_t;
 
 typedef enum glusterd_op_ {
         GD_OP_NONE = 0,
@@ -80,68 +98,96 @@ typedef enum glusterd_op_ {
         GD_OP_LIST_VOLUME,
         GD_OP_CLEARLOCKS_VOLUME,
         GD_OP_DEFRAG_BRICK_VOLUME,
+        GD_OP_COPY_FILE,
+        GD_OP_SYS_EXEC,
+        GD_OP_GSYNC_CREATE,
         GD_OP_MAX,
 } glusterd_op_t;
 
-
-struct glusterd_store_iter_ {
-        int     fd;
-        FILE    *file;
-        char    filepath[PATH_MAX];
-};
-
-typedef struct glusterd_store_iter_     glusterd_store_iter_t;
+extern const char * gd_op_list[];
 
 struct glusterd_volgen {
         dict_t *dict;
 };
+
 typedef struct {
-        struct rpc_clnt         *rpc;
-        gf_boolean_t            running;
+        struct rpc_clnt  *rpc;
+        gf_boolean_t      online;
 } nodesrv_t;
 
 typedef struct {
-        struct _volfile_ctx *volfile;
-	pthread_mutex_t   mutex;
-	struct list_head  peers;
-//	struct list_head  pending_peers;
-        gf_boolean_t      verify_volfile_checksum;
-        gf_boolean_t      trace;
-        uuid_t            uuid;
-        char              workdir[PATH_MAX];
-        rpcsvc_t          *rpc;
-        nodesrv_t         *shd;
-        struct pmap_registry *pmap;
-        struct list_head  volumes;
-        struct list_head  xprt_list;
-        glusterd_store_handle_t *handle;
-        gf_timer_t *timer;
-        glusterd_sm_tr_log_t op_sm_log;
+        gf_boolean_t    quorum;
+        double          quorum_ratio;
+        uint64_t        gl_opt_version;
+} gd_global_opts_t;
+
+typedef struct {
+        struct _volfile_ctx     *volfile;
+        pthread_mutex_t          mutex;
+        struct list_head         peers;
+        struct list_head         xaction_peers;
+        gf_boolean_t             verify_volfile_checksum;
+        gf_boolean_t             trace;
+        uuid_t                   uuid;
+        char                     workdir[PATH_MAX];
+        rpcsvc_t                *rpc;
+        nodesrv_t               *shd;
+        nodesrv_t               *nfs;
+        nodesrv_t               *quotad;
+        struct pmap_registry    *pmap;
+        struct list_head         volumes;
+        pthread_mutex_t          xprt_lock;
+        struct list_head         xprt_list;
+        gf_store_handle_t       *handle;
+        gf_timer_t              *timer;
+        glusterd_sm_tr_log_t     op_sm_log;
         struct rpc_clnt_program *gfs_mgmt;
-        struct list_head mount_specs;
-#ifdef DEBUG
+        dict_t                  *vol_lock; /* Dict for saving vol locks */
+        dict_t                  *glusterd_txn_opinfo; /* Dict for saving
+                                                       * transaction opinfos */
+        uuid_t                   global_txn_id; /* To be used in
+                                                 * heterogeneous
+                                                 * cluster with no
+                                                 * transaction ids */
+
+        struct list_head  mount_specs;
         gf_boolean_t      valgrind;
-#endif
+        pthread_t         brick_thread;
+        void             *hooks_priv;
+
+        /* need for proper handshake_t */
+        int           op_version; /* Starts with 1 for 3.3.0 */
+        xlator_t     *xl;       /* Should be set to 'THIS' before creating thread */
+        gf_boolean_t  pending_quorum_action;
+        dict_t       *opts;
+        synclock_t    big_lock;
+        gf_boolean_t  restart_done;
+        rpcsvc_t     *uds_rpc;  /* RPCSVC for the unix domain socket */
+        uint32_t      base_port;
 } glusterd_conf_t;
 
+
 typedef enum gf_brick_status {
         GF_BRICK_STOPPED,
         GF_BRICK_STARTED,
 } gf_brick_status_t;
 
 struct glusterd_brickinfo {
-        char    hostname[1024];
-        char    path[PATH_MAX];
-        struct list_head  brick_list;
-        uuid_t  uuid;
-        int     port;
-        int     rdma_port;
-        char   *logfile;
-        gf_boolean_t signed_in;
-        glusterd_store_handle_t *shandle;
-        gf_brick_status_t status;
-        struct rpc_clnt *rpc;
-        int decommissioned;
+        char               hostname[1024];
+        char               path[PATH_MAX];
+        char               brick_id[1024];/*Client xlator name, AFR changelog name*/
+        struct list_head   brick_list;
+        uuid_t             uuid;
+        int                port;
+        int                rdma_port;
+        char              *logfile;
+        gf_boolean_t       signed_in;
+        gf_store_handle_t *shandle;
+        gf_brick_status_t  status;
+        struct rpc_clnt   *rpc;
+        int                decommissioned;
+        char vg[PATH_MAX]; /* FIXME: Use max size for length of vg */
+        int     caps; /* Capability */
 };
 
 typedef struct glusterd_brickinfo glusterd_brickinfo_t;
@@ -152,9 +198,6 @@ struct gf_defrag_brickinfo_ {
         int   size;
 };
 
-struct glusterd_volinfo_;
-typedef struct glusterd_volinfo_ glusterd_volinfo_t;
-
 typedef int (*defrag_cbk_fn_t) (glusterd_volinfo_t *volinfo,
                                 gf_defrag_status_t status);
 
@@ -162,11 +205,12 @@ struct glusterd_defrag_info_ {
         uint64_t                     total_files;
         uint64_t                     total_data;
         uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
         gf_lock_t                    lock;
         int                          cmd;
         pthread_t                    th;
         gf_defrag_status_t           defrag_status;
-        struct rpc_clnt            * rpc;
+        struct rpc_clnt             *rpc;
         uint32_t                     connected;
         char                         mount[1024];
         char                         databuf[131072];
@@ -199,50 +243,91 @@ struct _auth {
 
 typedef struct _auth auth_t;
 
+/* Capabilities of xlator */
+#define CAPS_BD               0x00000001
+#define CAPS_THIN             0x00000002
+#define CAPS_OFFLOAD_COPY     0x00000004
+#define CAPS_OFFLOAD_SNAPSHOT 0x00000008
+#define CAPS_OFFLOAD_ZERO     0x00000020
+
+struct glusterd_rebalance_ {
+        gf_defrag_status_t       defrag_status;
+        uint64_t                 rebalance_files;
+        uint64_t                 rebalance_data;
+        uint64_t                 lookedup_files;
+        uint64_t                 skipped_files;
+        glusterd_defrag_info_t  *defrag;
+        gf_cli_defrag_type       defrag_cmd;
+        uint64_t                 rebalance_failures;
+        uuid_t                   rebalance_id;
+        double                   rebalance_time;
+        glusterd_op_t            op;
+        dict_t                  *dict; /* Dict to store misc information
+                                        * like list of bricks being removed */
+};
+
+typedef struct glusterd_rebalance_ glusterd_rebalance_t;
+
+struct glusterd_replace_brick_ {
+        gf_rb_status_t          rb_status;
+        glusterd_brickinfo_t   *src_brick;
+        glusterd_brickinfo_t   *dst_brick;
+        uuid_t                  rb_id;
+};
+
+typedef struct glusterd_replace_brick_ glusterd_replace_brick_t;
+
 struct glusterd_volinfo_ {
-        char                    volname[GLUSTERD_MAX_VOLUME_NAME];
-        int                     type;
-        int                     brick_count;
-        struct list_head        vol_list;
-        struct list_head        bricks;
-        glusterd_volume_status  status;
-        int                     sub_count;  /* backward compatibility */
-        int                     stripe_count;
-        int                     replica_count;
-        int                     dist_leaf_count; /* Number of bricks in one
+        char                      volname[GLUSTERD_MAX_VOLUME_NAME];
+        int                       type;
+        int                       brick_count;
+        struct list_head          vol_list;
+        struct list_head          bricks;
+        glusterd_volume_status    status;
+        int                       sub_count;  /* backward compatibility */
+        int                       stripe_count;
+        int                       replica_count;
+        int                       subvol_count; /* Number of subvolumes in a
+                                                 distribute volume */
+        int                       dist_leaf_count; /* Number of bricks in one
                                                     distribute subvolume */
         int                     port;
-        glusterd_store_handle_t *shandle;
-        glusterd_store_handle_t *rb_shandle;
+        gf_store_handle_t *shandle;
+        gf_store_handle_t *rb_shandle;
+        gf_store_handle_t *node_state_shandle;
+        gf_store_handle_t *quota_conf_shandle;
 
         /* Defrag/rebalance related */
-        gf_defrag_status_t      defrag_status;
-        uint64_t                rebalance_files;
-        uint64_t                rebalance_data;
-        uint64_t                lookedup_files;
-        glusterd_defrag_info_t  *defrag;
-        gf_cli_defrag_type      defrag_cmd;
+        glusterd_rebalance_t      rebal;
 
         /* Replace brick status */
-        gf_rb_status_t          rb_status;
-        glusterd_brickinfo_t    *src_brick;
-        glusterd_brickinfo_t    *dst_brick;
+        glusterd_replace_brick_t  rep_brick;
 
         int                     version;
+        uint32_t                quota_conf_version;
         uint32_t                cksum;
+        uint32_t                quota_conf_cksum;
         gf_transport_type       transport_type;
         gf_transport_type   nfs_transport_type;
 
-        dict_t                  *dict;
+        dict_t                   *dict;
+
+        uuid_t                    volume_id;
+        auth_t                    auth;
+        char                     *logdir;
+
+        dict_t                   *gsync_slaves;
 
-        uuid_t                  volume_id;
-        auth_t                  auth;
-        char                    *logdir;
+        int                       decommission_in_progress;
+        xlator_t                 *xl;
 
-        dict_t                  *gsync_slaves;
+        gf_boolean_t              memory_accounting;
+        int                      caps; /* Capability */
 
-        int                      decommission_in_progress;
-        xlator_t                *xl;
+        int                       op_version;
+        int                       client_op_version;
+        pthread_mutex_t           reflock;
+        int                       refcnt;
 };
 
 typedef enum gd_node_type_ {
@@ -250,6 +335,8 @@ typedef enum gd_node_type_ {
         GD_NODE_BRICK,
         GD_NODE_SHD,
         GD_NODE_REBALANCE,
+        GD_NODE_NFS,
+        GD_NODE_QUOTAD,
 } gd_node_type;
 
 typedef struct glusterd_pending_node_ {
@@ -259,6 +346,13 @@ typedef struct glusterd_pending_node_ {
         int32_t index;
 } glusterd_pending_node_t;
 
+struct gsync_config_opt_vals_ {
+        char           *op_name;
+        int             no_of_pos_vals;
+        gf_boolean_t    case_sensitive;
+        char           *values[GEO_CONF_MAX_OPT_VALS];
+};
+
 enum glusterd_op_ret {
         GLUSTERD_CONNECTION_AWAITED = 100,
 };
@@ -270,15 +364,19 @@ enum glusterd_vol_comp_status_ {
         GLUSTERD_VOL_COMP_RJT,
 };
 
-#define GLUSTERD_DEFAULT_WORKDIR "/etc/glusterd"
+#define GLUSTERD_DEFAULT_WORKDIR "/var/lib/glusterd"
 #define GLUSTERD_DEFAULT_PORT    GF_DEFAULT_BASE_PORT
 #define GLUSTERD_INFO_FILE      "glusterd.info"
+#define GLUSTERD_VOLUME_QUOTA_CONFIG "quota.conf"
 #define GLUSTERD_VOLUME_DIR_PREFIX "vols"
 #define GLUSTERD_PEER_DIR_PREFIX "peers"
 #define GLUSTERD_VOLUME_INFO_FILE "info"
 #define GLUSTERD_VOLUME_RBSTATE_FILE "rbstate"
 #define GLUSTERD_BRICK_INFO_DIR "bricks"
 #define GLUSTERD_CKSUM_FILE "cksum"
+#define GLUSTERD_VOL_QUOTA_CKSUM_FILE "quota.cksum"
+#define GLUSTERD_TRASH "trash"
+#define GLUSTERD_NODE_STATE_FILE "node_state.info"
 
 /* definitions related to replace brick */
 #define RB_CLIENT_MOUNTPOINT    "rb_mount"
@@ -303,6 +401,13 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
 #define GLUSTERD_GET_NFS_DIR(path, priv) \
         snprintf (path, PATH_MAX, "%s/nfs", priv->workdir);
 
+#define GLUSTERD_GET_QUOTAD_DIR(path, priv) \
+        snprintf (path, PATH_MAX, "%s/quotad", priv->workdir);
+
+#define GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH(abspath, volname, path)      \
+        snprintf (abspath, sizeof (abspath)-1,                          \
+                  DEFAULT_VAR_RUN_DIRECTORY"/%s%s", volname, path);
+
 #define GLUSTERD_REMOVE_SLASH_FROM_PATH(path,string) do {               \
                 int i = 0;                                              \
                 for (i = 1; i < strlen (path); i++) {                   \
@@ -312,22 +417,29 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                 }                                                       \
         } while (0)
 
-#define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volpath,hostname,brickpath) { \
-                char exp_path[PATH_MAX] = {0,};                         \
-                GLUSTERD_REMOVE_SLASH_FROM_PATH (brickpath, exp_path);  \
-                snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid",        \
-                          volpath, hostname, exp_path);                 \
-        }
+#define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volinfo,brickinfo, priv) do {      \
+                char exp_path[PATH_MAX] = {0,};                               \
+                char volpath[PATH_MAX]  = {0,};                               \
+                GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv);             \
+                GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);  \
+                snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid",              \
+                          volpath, brickinfo->hostname, exp_path);            \
+        } while (0)
 
 #define GLUSTERD_GET_NFS_PIDFILE(pidfile,nfspath) {                     \
                 snprintf (pidfile, PATH_MAX, "%s/run/nfs.pid",          \
                           nfspath);                                     \
         }
 
+#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile,quotadpath) {                     \
+                snprintf (pidfile, PATH_MAX, "%s/run/quotad.pid",          \
+                          quotadpath);                                     \
+        }
+
 #define GLUSTERD_STACK_DESTROY(frame) do {\
-		frame->local = NULL;	  \
-		STACK_DESTROY (frame->root);\
-	} while (0)
+                frame->local = NULL;                                    \
+                STACK_DESTROY (frame->root);                            \
+        } while (0)
 
 #define GLUSTERD_GET_DEFRAG_DIR(path, volinfo, priv) do {               \
                 char vol_path[PATH_MAX];                                \
@@ -335,30 +447,68 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                 snprintf (path, PATH_MAX, "%s/rebalance",vol_path);     \
         } while (0)
 
-#define GLUSTERD_GET_DEFRAG_SOCK_FILE(path, volinfo, priv) do {         \
+#define GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD(path, volinfo, priv) do {     \
                 char defrag_path[PATH_MAX];                             \
                 GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);    \
                 snprintf (path, PATH_MAX, "%s/%s.sock", defrag_path,    \
-                           uuid_utoa(priv->uuid));                      \
+                           uuid_utoa(MY_UUID));                         \
+        } while (0)
+
+#define GLUSTERD_GET_DEFRAG_SOCK_FILE(path, volinfo) do {                   \
+                snprintf (path, UNIX_PATH_MAX, DEFAULT_VAR_RUN_DIRECTORY    \
+                          "/gluster-rebalance-%s.sock",                     \
+                           uuid_utoa(volinfo->volume_id));                  \
         } while (0)
 
 #define GLUSTERD_GET_DEFRAG_PID_FILE(path, volinfo, priv) do {          \
                 char defrag_path[PATH_MAX];                             \
                 GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);    \
                 snprintf (path, PATH_MAX, "%s/%s.pid", defrag_path,     \
-                           uuid_utoa(priv->uuid));                      \
+                           uuid_utoa(MY_UUID));                         \
         } while (0)
 
+#define GLUSTERFS_GET_AUX_MOUNT_PIDFILE(pidfile, volname) {               \
+                snprintf (pidfile, PATH_MAX-1,                            \
+                          DEFAULT_VAR_RUN_DIRECTORY"/%s.pid", volname);   \
+        }
+
+int glusterd_uuid_init();
+
+int glusterd_uuid_generate_save ();
+
+#define MY_UUID (__glusterd_uuid())
+
+static inline unsigned char *
+__glusterd_uuid()
+{
+        glusterd_conf_t *priv = THIS->private;
+
+        if (uuid_is_null (priv->uuid))
+                glusterd_uuid_init();
+        return &priv->uuid[0];
+}
+
+int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
+                                rpc_clnt_event_t event,
+                                void *data, rpc_clnt_notify_t notify_fn);
+
+int
+glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe, fop_cbk_fn_t fn);
+
+int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn);
 
 int32_t
 glusterd_brick_from_brickinfo (glusterd_brickinfo_t *brickinfo,
                                char **new_brick);
 int
-glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port);
+glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
+                      dict_t *dict);
 
 int
-glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *hostname,
-                               int port, int32_t op_ret, int32_t op_errno);
+glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
+                               char *remote_hostname, int port, int32_t op_ret,
+                               int32_t op_errno);
 
 int
 glusterd_friend_find (uuid_t uuid, char *hostname,
@@ -371,6 +521,9 @@ glusterd_friend_add (const char *hoststr, int port,
                      gf_boolean_t restore, glusterd_peerctx_args_t *args);
 
 int
+glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                            glusterd_peerctx_args_t *args);
+int
 glusterd_friend_remove (uuid_t uuid, char *hostname);
 
 int
@@ -380,6 +533,14 @@ int
 glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status);
 
 int
+glusterd_op_volume_lock_send_resp (rpcsvc_request_t *req,
+                                   uuid_t *txn_id, int32_t status);
+
+int
+glusterd_op_volume_unlock_send_resp (rpcsvc_request_t *req,
+                                     uuid_t *txn_id, int32_t status);
+
+int
 glusterd_op_stage_send_resp (rpcsvc_request_t *req,
                              int32_t op, int32_t status,
                              char *op_errstr, dict_t *rsp_dict);
@@ -423,7 +584,8 @@ glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req);
 
 int
 glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *hostname, int port);
+                              int32_t op_errno, char *op_errstr, char *hostname,
+                              int port, dict_t *dict);
 
 int
 glusterd_op_commit_send_resp (rpcsvc_request_t *req,
@@ -435,7 +597,7 @@ glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int por
 
 int
 glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                        uuid_t uuid);
+                        uuid_t uuid, dict_t *dict);
 
 int
 glusterd_handle_cli_deprobe (rpcsvc_request_t *req);
@@ -510,6 +672,12 @@ int
 glusterd_handle_reset_volume (rpcsvc_request_t *req);
 
 int
+glusterd_handle_copy_file (rpcsvc_request_t *req);
+
+int
+glusterd_handle_sys_exec (rpcsvc_request_t *req);
+
+int
 glusterd_handle_gsync_set (rpcsvc_request_t *req);
 
 int
@@ -520,7 +688,8 @@ glusterd_handle_fsm_log (rpcsvc_request_t *req);
 
 int
 glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                                int32_t op_errno, char *hostname);
+                                int32_t op_errno, char *op_errstr,
+                                char *hostname, dict_t *dict);
 
 int
 glusterd_fetchspec_notify (xlator_t *this);
@@ -537,8 +706,8 @@ glusterd_volume_txn (rpcsvc_request_t *req, char *volname, int flags,
                      glusterd_op_t op);
 
 int
-glusterd_peer_handshake (xlator_t *this, struct rpc_clnt *rpc,
-                         glusterd_peerctx_t *peerctx);
+glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
+                            glusterd_peerctx_t *peerctx);
 
 int
 glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict, char **op_errstr);
@@ -559,8 +728,8 @@ glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
                           rpc_clnt_event_t event, void *data);
 
 int
-glusterd_shd_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event, void *data);
+glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                             rpc_clnt_event_t event, void *data);
 
 int
 glusterd_rpc_create (struct rpc_clnt **rpc, dict_t *options,
@@ -568,7 +737,8 @@ glusterd_rpc_create (struct rpc_clnt **rpc, dict_t *options,
 
 
 /* handler functions */
-int32_t glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx);
+int32_t glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                           char *err_str, size_t size);
 
 /* removed other definitions as they have been defined elsewhere in this file*/
 
@@ -576,7 +746,12 @@ int glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req);
 int glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req);
 
 int glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
-                                  size_t len, int cmd, defrag_cbk_fn_t cbk);
+                                  size_t len, int cmd, defrag_cbk_fn_t cbk,
+                                  glusterd_op_t op);
+int
+glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
+                               gf_boolean_t reconnect);
+
 int glusterd_handle_cli_heal_volume (rpcsvc_request_t *req);
 
 int glusterd_handle_cli_list_volume (rpcsvc_request_t *req);
@@ -586,8 +761,14 @@ int glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr);
 int glusterd_op_heal_volume (dict_t *dict, char **op_errstr);
 int glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr);
 int glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-int glusterd_op_quota (dict_t *dict, char **op_errstr);
-int glusterd_op_stage_quota (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr);
+int glusterd_op_copy_file (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr);
+int glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr);
+int glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
 int glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                                      dict_t *rsp_dict);
 int glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict);
@@ -614,7 +795,8 @@ int glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr);
 int glusterd_op_statedump_volume (dict_t *dict, char **op_errstr);
 
 int glusterd_op_stage_clearlocks_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr,
+                                   dict_t *rsp_dict);
 
 /* misc */
 void glusterd_do_replace_brick (void *data);
@@ -624,4 +806,21 @@ int glusterd_op_stop_volume_args_get (dict_t *dict, char** volname, int *flags);
 int glusterd_op_statedump_volume_args_get (dict_t *dict, char **volname,
                                            char **options, int *option_cnt);
 
+int glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
+                                char **master, char **slave, char **host_uuid);
+/* Synctask part */
+int32_t glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
+                                    void *dict);
+int32_t
+glusterd_defrag_event_notify_handle (dict_t *dict);
+
+int32_t
+glusterd_txn_opinfo_dict_init ();
+
+void
+glusterd_txn_opinfo_dict_fini ();
+
+void
+glusterd_txn_opinfo_init ();
+
 #endif
diff --git a/xlators/mount/fuse/src/Makefile.am b/xlators/mount/fuse/src/Makefile.am
index 7bb3931ec..653121d18 100644
--- a/xlators/mount/fuse/src/Makefile.am
+++ b/xlators/mount/fuse/src/Makefile.am
@@ -1,26 +1,36 @@
-noinst_HEADERS = $(CONTRIBDIR)/fuse-include/fuse_kernel.h\
-	$(CONTRIBDIR)/fuse-include/fuse-mount.h\
+noinst_HEADERS_linux = $(CONTRIBDIR)/fuse-include/fuse_kernel.h\
+	$(CONTRIBDIR)/fuse-include/mount_util.h\
+	$(CONTRIBDIR)/fuse-lib/mount-gluster-compat.h
+noinst_HEADERS_darwin =	$(CONTRIBDIR)/fuse-include/fuse_kernel_macfuse.h
+noinst_HEADERS_common = $(CONTRIBDIR)/fuse-include/fuse-mount.h\
 	$(CONTRIBDIR)/fuse-include/fuse-misc.h fuse-mem-types.h \
 	fuse-bridge.h
 
+if GF_DARWIN_HOST_OS
+    noinst_HEADERS = $(noinst_HEADERS_common) $(noinst_HEADERS_darwin)
+else
+    noinst_HEADERS = $(noinst_HEADERS_common) $(noinst_HEADERS_linux)
+endif
+
 xlator_LTLIBRARIES = fuse.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
 
 if GF_DARWIN_HOST_OS
     mount_source=$(CONTRIBDIR)/macfuse/mount_darwin.c
 else
-    mount_source=$(CONTRIBDIR)/fuse-lib/mount.c
+    mount_source=$(CONTRIBDIR)/fuse-lib/mount.c $(CONTRIBDIR)/fuse-lib/mount-common.c
 endif
 
 fuse_la_SOURCES = fuse-helpers.c fuse-resolve.c fuse-bridge.c \
 	$(CONTRIBDIR)/fuse-lib/misc.c $(mount_source)
 
-fuse_la_LDFLAGS = -module -avoidversion -shared -nostartfiles
+fuse_la_LDFLAGS = -module -avoid-version
 fuse_la_LIBADD = @GF_FUSE_LDADD@
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) -Wall \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/fuse-include \
-	 $(GF_CFLAGS) $(GF_FUSE_CFLAGS)
+	-I$(CONTRIBDIR)/fuse-lib $(GF_FUSE_CFLAGS)
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 3f3962cbc..d9055468e 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -1,31 +1,100 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
+#include <sys/wait.h>
 #include "fuse-bridge.h"
+#include "mount-gluster-compat.h"
+#include "glusterfs.h"
+#include "glusterfs-acl.h"
+
+#ifdef __NetBSD__
+#undef open /* in perfuse.h, pulled from mount-gluster-compat.h */
+#endif
 
 static int gf_fuse_conn_err_log;
 static int gf_fuse_xattr_enotsup_log;
 
 void fini (xlator_t *this_xl);
 
+static void fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino);
+
+/*
+ * Send an invalidate notification up to fuse to purge the file from local
+ * page cache.
+ */
+static int32_t
+fuse_invalidate(xlator_t *this, inode_t *inode)
+{
+        fuse_private_t *priv = this->private;
+        uint64_t nodeid;
+
+        /*
+         * NOTE: We only invalidate at the moment if fopen_keep_cache is
+         * enabled because otherwise this is a departure from default
+         * behavior. Specifically, the performance/write-behind xlator
+         * causes unconditional invalidations on write requests.
+         */
+        if (!priv->fopen_keep_cache)
+                return 0;
+
+        nodeid = inode_to_fuse_nodeid(inode);
+        gf_log(this->name, GF_LOG_DEBUG, "Invalidate inode id %lu.", nodeid);
+        fuse_log_eh (this, "Sending invalidate inode id: %lu gfid: %s", nodeid,
+                     uuid_utoa (inode->gfid));
+        fuse_invalidate_inode(this, nodeid);
+
+        return 0;
+}
+
+static int32_t
+fuse_forget_cbk (xlator_t *this, inode_t *inode)
+{
+        //Nothing to free in inode ctx, hence return.
+        return 0;
+}
+
+void
+fuse_inode_set_need_lookup (inode_t *inode, xlator_t *this)
+{
+	uint64_t          need_lookup = 1;
+
+	if (!inode || !this)
+		return;
+
+	inode_ctx_set (inode, this, &need_lookup);
+
+	return;
+}
+
+
+gf_boolean_t
+fuse_inode_needs_lookup (inode_t *inode, xlator_t *this)
+{
+	uint64_t          need_lookup = 0;
+	gf_boolean_t      ret = _gf_false;
+
+	if (!inode || !this)
+		return ret;
+
+	inode_ctx_get (inode, this, &need_lookup);
+	if (need_lookup)
+		ret = _gf_true;
+	need_lookup = 0;
+	inode_ctx_set (inode, this, &need_lookup);
+
+	return ret;
+}
+
+
 fuse_fd_ctx_t *
-__fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
+__fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
 {
         uint64_t       val    = 0;
         int32_t        ret    = 0;
@@ -38,7 +107,9 @@ __fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
         if (fd_ctx == NULL) {
                 fd_ctx = GF_CALLOC (1, sizeof (*fd_ctx),
                                     gf_fuse_mt_fd_ctx_t);
-
+                if (!fd_ctx) {
+                    goto out;
+                }
                 ret = __fd_ctx_set (fd, this,
                                     (uint64_t)(unsigned long)fd_ctx);
                 if (ret < 0) {
@@ -48,12 +119,12 @@ __fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
                         fd_ctx = NULL;
                 }
         }
-
+out:
         return fd_ctx;
 }
 
 fuse_fd_ctx_t *
-fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
+fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
 {
         fuse_fd_ctx_t *fd_ctx = NULL;
 
@@ -63,7 +134,7 @@ fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this)
 
         LOCK (&fd->lock);
         {
-                fd_ctx = __fuse_fd_ctx_check_n_create (fd, this);
+                fd_ctx = __fuse_fd_ctx_check_n_create (this, fd);
         }
         UNLOCK (&fd->lock);
 
@@ -71,6 +142,23 @@ out:
         return fd_ctx;
 }
 
+fuse_fd_ctx_t *
+fuse_fd_ctx_get (xlator_t *this, fd_t *fd)
+{
+        fuse_fd_ctx_t *fdctx = NULL;
+        uint64_t       value = 0;
+        int            ret   = 0;
+
+        ret = fd_ctx_get (fd, this, &value);
+        if (ret < 0) {
+                goto out;
+        }
+
+        fdctx = (fuse_fd_ctx_t *) (unsigned long)value;
+
+out:
+        return fdctx;
+}
 
 /*
  * iov_out should contain a fuse_out_header at zeroth position.
@@ -86,7 +174,7 @@ send_fuse_iov (xlator_t *this, fuse_in_header_t *finh, struct iovec *iov_out,
 
         if (!this || !finh || !iov_out) {
                 gf_log ("send_fuse_iov", GF_LOG_ERROR,"Invalid arguments");
-                return -1;
+                return EINVAL;
         }
         priv = this->private;
 
@@ -141,7 +229,7 @@ send_fuse_data (xlator_t *this, fuse_in_header_t *finh, void *data, size_t size)
 
 
 static void
-fuse_invalidate (xlator_t *this, uint64_t fuse_ino)
+fuse_invalidate_entry (xlator_t *this, uint64_t fuse_ino)
 {
         struct fuse_out_header             *fouh   = NULL;
         struct fuse_notify_inval_entry_out *fnieo  = NULL;
@@ -150,8 +238,7 @@ fuse_invalidate (xlator_t *this, uint64_t fuse_ino)
         inode_t                            *inode  = NULL;
         size_t                              nlen   = 0;
         int                                 rv     = 0;
-
-        char inval_buf[INVAL_BUF_SIZE] = {0,};
+        char inval_buf[INVAL_BUF_SIZE]             = {0,};
 
         fouh  = (struct fuse_out_header *)inval_buf;
         fnieo = (struct fuse_notify_inval_entry_out *)(fouh + 1);
@@ -184,7 +271,73 @@ fuse_invalidate (xlator_t *this, uint64_t fuse_ino)
 
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE entry: "
                         "%"PRIu64"/%s", fnieo->parent, dentry->name);
+
+                if (dentry->parent) {
+                        fuse_log_eh (this, "Invalidated entry %s (parent: %s)",
+                                     dentry->name,
+                                     uuid_utoa (dentry->parent->gfid));
+                } else {
+                        fuse_log_eh (this, "Invalidated entry %s(nodeid: %ld)",
+                                     dentry->name, fnieo->parent);
+                }
         }
+
+        if (inode)
+                inode_unref (inode);
+}
+
+/*
+ * Send an inval inode notification to fuse. This causes an invalidation of the
+ * entire page cache mapping on the inode.
+ */
+static void
+fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino)
+{
+        struct fuse_out_header *fouh = NULL;
+        struct fuse_notify_inval_inode_out *fniio = NULL;
+        fuse_private_t *priv = NULL;
+        int rv = 0;
+        char inval_buf[INVAL_BUF_SIZE] = {0};
+        inode_t    *inode = NULL;
+
+        fouh = (struct fuse_out_header *) inval_buf;
+        fniio = (struct fuse_notify_inval_inode_out *) (fouh + 1);
+
+        priv = this->private;
+
+        if (priv->revchan_out < 0)
+                return;
+
+        fouh->unique = 0;
+        fouh->error = FUSE_NOTIFY_INVAL_INODE;
+        fouh->len = sizeof(struct fuse_out_header) +
+                sizeof(struct fuse_notify_inval_inode_out);
+
+        /* inval the entire mapping until we learn how to be more granular */
+        fniio->ino = fuse_ino;
+        fniio->off = 0;
+        fniio->len = -1;
+
+        inode = fuse_ino_to_inode (fuse_ino, this);
+
+        rv = write(priv->revchan_out, inval_buf, fouh->len);
+        if (rv != fouh->len) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR, "kernel notification "
+                        "daemon defunct");
+                close(priv->fd);
+        }
+
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE inode: %lu", fuse_ino);
+
+        if (inode) {
+                fuse_log_eh (this, "Invalidated inode %lu (gfid: %s)",
+                             fuse_ino, uuid_utoa (inode->gfid));
+        } else {
+                fuse_log_eh (this, "Invalidated inode %lu ", fuse_ino);
+        }
+
+        if (inode)
+                inode_unref (inode);
 }
 
 int
@@ -192,18 +345,36 @@ send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error)
 {
         struct fuse_out_header fouh = {0, };
         struct iovec iov_out;
+        inode_t  *inode = NULL;
 
         fouh.error = -error;
         iov_out.iov_base = &fouh;
 
+        inode = fuse_ino_to_inode (finh->nodeid, this);
+
+        // filter out ENOENT
+        if (error != ENOENT) {
+                if (inode) {
+                        fuse_log_eh (this,"Sending %s for operation %d on "
+                                     "inode %s", strerror (error), finh->opcode,
+                                     uuid_utoa (inode->gfid));
+                } else {
+                        fuse_log_eh (this, "Sending %s for operation %d on "
+                                     "inode %ld", strerror (error),
+                                     finh->opcode, finh->nodeid);
+                }
+        }
+
+        if (inode)
+                inode_unref (inode);
+
         return send_fuse_iov (this, finh, &iov_out, 1);
 }
 
-
 static int
 fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
-                inode_t *inode, struct iatt *buf)
+                inode_t *inode, struct iatt *buf, dict_t *xdata)
 {
         fuse_state_t          *state        = NULL;
         fuse_in_header_t      *finh         = NULL;
@@ -215,18 +386,39 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
-        if (!op_ret && __is_root_gfid (state->loc.inode->gfid)) {
-                buf->ia_ino = 1;
+        if (op_ret == 0) {
+                if (__is_root_gfid (state->loc.inode->gfid))
+                        buf->ia_ino = 1;
+                if (uuid_is_null (buf->ia_gfid)) {
+                        /* With a NULL gfid inode linking is
+                           not possible. Let's not pretend this
+                           call was a "success".
+                        */
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "Received NULL gfid for %s. Forcing EIO",
+                                state->loc.path);
+                        op_ret = -1;
+                        op_errno = EIO;
+                }
         }
 
+        /* log into the event-history after the null uuid check is done, since
+         * the op_ret and op_errno are being changed if the gfid is NULL.
+         */
+        fuse_log_eh (this, "op_ret: %d op_errno: %d "
+                     "%"PRIu64": %s() %s => %s", op_ret, op_errno,
+                     frame->root->unique, gf_fop_list[frame->root->op],
+                     state->loc.path, (op_ret == 0)?
+                     uuid_utoa(buf->ia_gfid):uuid_utoa(state->loc.gfid));
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRId64,
+                        "%"PRIu64": %s() %s => %"PRIu64,
                         frame->root->unique, gf_fop_list[frame->root->op],
                         state->loc.path, buf->ia_ino);
 
                 buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &feo.attr);
+                gf_fuse_stat2attr (buf, &feo.attr, priv->enable_ino32);
 
                 if (!buf->ia_ino) {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -270,7 +462,16 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path,
                         strerror (op_errno));
-                send_fuse_err (this, state->finh, op_errno);
+
+		if ((op_errno == ENOENT) && (priv->negative_timeout != 0)) {
+			feo.entry_valid =
+				calc_timeout_sec (priv->negative_timeout);
+			feo.entry_valid_nsec =
+				calc_timeout_nsec (priv->negative_timeout);
+			send_fuse_obj (this, finh, &feo);
+		} else {
+			send_fuse_err (this, state->finh, op_errno);
+                }
         }
 
         free_fuse_state (state);
@@ -278,18 +479,17 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int
 fuse_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
-        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, buf);
+        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, buf,
+                        xdata);
         return 0;
 }
 
-
 static int
 fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
@@ -305,6 +505,13 @@ fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (op_ret == -1 && state->is_revalidate == 1) {
                 itable = state->itable;
+		/*
+		 * A stale mapping might exist for a dentry/inode that has been
+		 * removed from another client.
+		 */
+		if (op_errno == ENOENT)
+			inode_unlink(state->loc.inode, state->loc.parent,
+				     state->loc.name);
                 inode_unref (state->loc.inode);
                 state->loc.inode = inode_new (itable);
                 state->is_revalidate = 2;
@@ -314,11 +521,12 @@ fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 STACK_WIND (frame, fuse_lookup_cbk,
                             prev->this, prev->this->fops->lookup,
-                            &state->loc, state->dict);
+                            &state->loc, state->xdata);
                 return 0;
         }
 
-        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, stat);
+        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, stat,
+                        dict);
         return 0;
 }
 
@@ -333,6 +541,14 @@ fuse_lookup_resume (fuse_state_t *state)
                 return;
         }
 
+        /* parent was resolved, entry could not, may be a missing gfid?
+         * Hence try to do a regular lookup
+         */
+        if ((state->resolve.op_ret == -1)
+            && (state->resolve.op_errno == ENODATA)) {
+                state->resolve.op_ret = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": LOOKUP %s(%s)", state->finh->unique,
@@ -343,17 +559,20 @@ fuse_lookup_resume (fuse_state_t *state)
                         "%"PRIu64": LOOKUP %s", state->finh->unique,
                         state->loc.path);
                 state->loc.inode = inode_new (state->loc.parent->table);
+                if (uuid_is_null (state->gfid))
+                        uuid_generate (state->gfid);
+                fuse_gfid_set (state);
         }
 
         FUSE_FOP (state, fuse_lookup_cbk, GF_FOP_LOOKUP,
-                  lookup, &state->loc, state->dict);
+                  lookup, &state->loc, state->xdata);
 }
 
 static void
 fuse_lookup (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        char         *name      = msg;
-        fuse_state_t *state     = NULL;
+        char           *name     = msg;
+        fuse_state_t   *state    = NULL;
 
         GET_STATE (this, finh, state);
 
@@ -361,16 +580,27 @@ fuse_lookup (xlator_t *this, fuse_in_header_t *finh, void *msg)
                                         finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_lookup_resume);
+
+        return;
 }
 
+static inline void
+do_forget(xlator_t *this, uint64_t unique, uint64_t nodeid, uint64_t nlookup)
+{
+	inode_t *fuse_inode = fuse_ino_to_inode(nodeid, this);
+
+	fuse_log_eh(this, "%"PRIu64": FORGET %"PRIu64"/%"PRIu64" gfid: (%s)",
+		    unique, nodeid, nlookup, uuid_utoa(fuse_inode->gfid));
+
+	inode_forget(fuse_inode, nlookup);
+	inode_unref(fuse_inode);
+}
 
 static void
 fuse_forget (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
 {
-        struct fuse_forget_in *ffi = msg;
-
-        inode_t      *fuse_inode;
+        struct fuse_forget_in *ffi        = msg;
 
         if (finh->nodeid == 1) {
                 GF_FREE (finh);
@@ -381,19 +611,35 @@ fuse_forget (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 "%"PRIu64": FORGET %"PRIu64"/%"PRIu64,
                 finh->unique, finh->nodeid, ffi->nlookup);
 
-        fuse_inode = fuse_ino_to_inode (finh->nodeid, this);
-
-        inode_forget (fuse_inode, ffi->nlookup);
-        inode_unref (fuse_inode);
+	do_forget(this, finh->unique, finh->nodeid, ffi->nlookup);
 
         GF_FREE (finh);
 }
 
+static void
+fuse_batch_forget(xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+	struct fuse_batch_forget_in *fbfi = msg;
+	struct fuse_forget_one *ffo = (struct fuse_forget_one *) (fbfi + 1);
+	int i;
+
+	gf_log("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": BATCH_FORGET %"PRIu64"/%"PRIu32,
+		finh->unique, finh->nodeid, fbfi->count);
+
+	for (i = 0; i < fbfi->count; i++) {
+                if (ffo[i].nodeid == 1)
+                        continue;
+		do_forget(this, finh->unique, ffo[i].nodeid, ffo[i].nlookup);
+        }
+
+	GF_FREE(finh);
+}
 
 static int
 fuse_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         fuse_state_t     *state;
         fuse_in_header_t *finh;
@@ -404,15 +650,17 @@ fuse_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRId64, frame->root->unique,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
                         gf_fop_list[frame->root->op],
                         state->loc.path ? state->loc.path : "ERR",
                         prebuf->ia_ino);
 
                 postbuf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (postbuf, &fao.attr);
+                gf_fuse_stat2attr (postbuf, &fao.attr, priv->enable_ino32);
 
                 fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
                 fao.attr_valid_nsec =
@@ -442,10 +690,9 @@ fuse_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int
 fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf)
+               int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         fuse_state_t     *state;
         fuse_in_header_t *finh;
@@ -456,15 +703,19 @@ fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() %s => "
+                   "gfid: %s", op_ret, op_errno, frame->root->unique,
+                   gf_fop_list[frame->root->op], state->loc.path,
+                   state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRId64, frame->root->unique,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
                         gf_fop_list[frame->root->op],
                         state->loc.path ? state->loc.path : "ERR",
                         buf->ia_ino);
 
                 buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &fao.attr);
+                gf_fuse_stat2attr (buf, &fao.attr, priv->enable_ino32);
 
                 fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
                 fao.attr_valid_nsec =
@@ -479,9 +730,9 @@ fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 send_fuse_obj (this, finh, &fao);
 #endif
         } else {
-                GF_LOG_OCCASIONALLY ( gf_fuse_conn_err_log, "glusterfs-fuse", 
-                                      GF_LOG_WARNING, 
-                                      "%"PRIu64": %s() %s => -1 (%s)", 
+                GF_LOG_OCCASIONALLY ( gf_fuse_conn_err_log, "glusterfs-fuse",
+                                      GF_LOG_WARNING,
+                                      "%"PRIu64": %s() %s => -1 (%s)",
                                       frame->root->unique,
                                       gf_fop_list[frame->root->op],
                                       state->loc.path ? state->loc.path : "ERR",
@@ -496,14 +747,13 @@ fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int
 fuse_root_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *stat, dict_t *dict,
                       struct iatt *postparent)
 {
-        fuse_attr_cbk (frame, cookie, this, op_ret, op_errno, stat);
+        fuse_attr_cbk (frame, cookie, this, op_ret, op_errno, stat, dict);
 
         return 0;
 }
@@ -515,15 +765,15 @@ fuse_getattr_resume (fuse_state_t *state)
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": GETATTR %"PRIu64" (%s) resolution failed",
                         state->finh->unique, state->finh->nodeid,
-			uuid_utoa (state->resolve.gfid));
+                        uuid_utoa (state->resolve.gfid));
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
         }
 
-	if (!IA_ISDIR (state->loc.inode->ia_type)) {
-		state->fd = fd_lookup (state->loc.inode, state->finh->pid);
-	}
+        if (!IA_ISDIR (state->loc.inode->ia_type)) {
+                state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+        }
 
         if (!state->fd) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -532,7 +782,7 @@ fuse_getattr_resume (fuse_state_t *state)
                         state->loc.path);
 
                 FUSE_FOP (state, fuse_attr_cbk, GF_FOP_STAT,
-                          stat, &state->loc);
+                          stat, &state->loc, state->xdata);
         } else {
 
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -541,7 +791,7 @@ fuse_getattr_resume (fuse_state_t *state)
                         state->loc.path, state->fd);
 
                 FUSE_FOP (state, fuse_attr_cbk, GF_FOP_FSTAT,
-                          fstat, state->fd);
+                          fstat, state->fd, state->xdata);
         }
 }
 
@@ -569,7 +819,7 @@ fuse_getattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 fuse_gfid_set (state);
 
                 FUSE_FOP (state, fuse_root_lookup_cbk, GF_FOP_LOOKUP,
-                          lookup, &state->loc, state->dict);
+                          lookup, &state->loc, state->xdata);
                 return;
         }
 
@@ -578,14 +828,12 @@ fuse_getattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_getattr_resume);
 }
 
-
 static int32_t
 fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
 {
         int32_t        ret    = 0;
         fuse_fd_ctx_t *fdctx  = NULL, *tmp_fdctx = NULL;
         fd_t          *tmp_fd = NULL;
-        uint64_t       val    = 0;
 
         GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", this, out, ret,
                                         -EINVAL);
@@ -594,7 +842,7 @@ fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
         GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", foo, out, ret,
                                         -EINVAL);
 
-        fdctx = fuse_fd_ctx_check_n_create (fd, this);
+        fdctx = fuse_fd_ctx_get (this, fd);
         if (!fdctx) {
                 ret = -ENOMEM;
                 goto out;
@@ -602,14 +850,11 @@ fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
 
         tmp_fd = fd_lookup (fd->inode, 0);
         if (tmp_fd) {
-                ret = fd_ctx_get (tmp_fd, this, &val);
-                if (!ret) {
-                        tmp_fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
-                        if (tmp_fdctx) {
-                                foo->open_flags &= ~FOPEN_DIRECT_IO;
-                                foo->open_flags |= (tmp_fdctx->open_flags
-                                                    & FOPEN_DIRECT_IO);
-                        }
+                tmp_fdctx = fuse_fd_ctx_get (this, tmp_fd);
+                if (tmp_fdctx) {
+                        foo->open_flags &= ~FOPEN_DIRECT_IO;
+                        foo->open_flags |= (tmp_fdctx->open_flags
+                                            & FOPEN_DIRECT_IO);
                 }
         }
 
@@ -624,10 +869,9 @@ out:
         return ret;
 }
 
-
 static int
 fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd)
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         fuse_state_t         *state  = NULL;
         fuse_in_header_t     *finh   = NULL;
@@ -639,6 +883,8 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 foo.fh = (uintptr_t) fd;
                 foo.open_flags = 0;
@@ -649,17 +895,27 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             || (priv->direct_io_mode == 1))
                                 foo.open_flags |= FOPEN_DIRECT_IO;
 #ifdef GF_DARWIN_HOST_OS
-                                /* In Linux: by default, buffer cache
-                                 * is purged upon open, setting
-                                 * FOPEN_KEEP_CACHE implies no-purge
-                                 *
-                                 * In MacFUSE: by default, buffer cache
-                                 * is left intact upon open, setting
-                                 * FOPEN_PURGE_UBC implies purge
-                                 *
-                                 * [[Interesting...]]
-                                 */
+                        /* In Linux: by default, buffer cache
+                         * is purged upon open, setting
+                         * FOPEN_KEEP_CACHE implies no-purge
+                         *
+                         * In MacFUSE: by default, buffer cache
+                         * is left intact upon open, setting
+                         * FOPEN_PURGE_UBC implies purge
+                         *
+                         * [[Interesting...]]
+                         */
+                        if (!priv->fopen_keep_cache)
                                 foo.open_flags |= FOPEN_PURGE_UBC;
+#else
+                        /*
+                         * If fopen-keep-cache is enabled, we set the associated
+                         * flag here such that files are not invalidated on open.
+                         * File invalidations occur either in fuse or explicitly
+                         * when the cache is set invalid on the inode.
+                         */
+                        if (priv->fopen_keep_cache)
+                                foo.open_flags |= FOPEN_KEEP_CACHE;
 #endif
                 }
 
@@ -671,8 +927,9 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (ret < 0) {
                         op_errno = -ret;
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "cannot inherit direct-io values from fds "
-                                "already opened");
+                                "cannot inherit direct-io values for fd "
+                                "(ptr:%p inode-gfid:%s) from fds already "
+                                "opened", fd, uuid_utoa (fd->inode->gfid));
                         goto err;
                 }
 
@@ -700,26 +957,24 @@ out:
         return 0;
 }
 
-
 static void
 fuse_do_truncate (fuse_state_t *state, size_t size)
 {
         if (state->fd) {
                 FUSE_FOP (state, fuse_truncate_cbk, GF_FOP_FTRUNCATE,
-                          ftruncate, state->fd, size);
+                          ftruncate, state->fd, size, state->xdata);
         } else {
                 FUSE_FOP (state, fuse_truncate_cbk, GF_FOP_TRUNCATE,
-                          truncate, &state->loc, size);
+                          truncate, &state->loc, size, state->xdata);
         }
 
         return;
 }
 
-
 static int
 fuse_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *statpre, struct iatt *statpost)
+                  struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
         fuse_state_t     *state;
         fuse_in_header_t *finh;
@@ -732,15 +987,20 @@ fuse_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh(this, "op_ret: %d, op_errno: %d, %"PRIu64", %s() %s => "
+                    "gfid: %s", op_ret, op_errno, frame->root->unique,
+                    gf_fop_list[frame->root->op], state->loc.path,
+                    state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRId64, frame->root->unique,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
                         gf_fop_list[frame->root->op],
                         state->loc.path ? state->loc.path : "ERR",
                         statpost->ia_ino);
 
                 statpost->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (statpost, &fao.attr);
+                gf_fuse_stat2attr (statpost, &fao.attr, priv->enable_ino32);
 
                 fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
                 fao.attr_valid_nsec =
@@ -779,7 +1039,6 @@ fuse_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int32_t
 fattr_to_gf_set_attr (int32_t valid)
 {
@@ -806,7 +1065,6 @@ fattr_to_gf_set_attr (int32_t valid)
         return gf_valid;
 }
 
-
 #define FATTR_MASK   (FATTR_SIZE                        \
                       | FATTR_UID | FATTR_GID           \
                       | FATTR_ATIME | FATTR_MTIME       \
@@ -819,7 +1077,7 @@ fuse_setattr_resume (fuse_state_t *state)
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": SETATTR %"PRIu64" (%s) resolution failed",
                         state->finh->unique, state->finh->nodeid,
-			uuid_utoa (state->resolve.gfid));
+                        uuid_utoa (state->resolve.gfid));
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -846,11 +1104,13 @@ fuse_setattr_resume (fuse_state_t *state)
 
                         FUSE_FOP (state, fuse_setattr_cbk, GF_FOP_FSETATTR,
                                   fsetattr, state->fd, &state->attr,
-                                  fattr_to_gf_set_attr (state->valid));
+                                  fattr_to_gf_set_attr (state->valid),
+                                  state->xdata);
                 } else {
                         FUSE_FOP (state, fuse_setattr_cbk, GF_FOP_SETATTR,
                                   setattr, &state->loc, &state->attr,
-                                  fattr_to_gf_set_attr (state->valid));
+                                  fattr_to_gf_set_attr (state->valid),
+                                  state->xdata);
                 }
         } else {
                 fuse_do_truncate (state, state->size);
@@ -873,10 +1133,10 @@ fuse_setattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 /* We need no loc if kernel sent us an fd and
                  * we are not fiddling with times */
                 state->fd = FH_TO_FD (fsi->fh);
-		fuse_resolve_fd_init (state, &state->resolve, state->fd);
-	} else {
-		fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
-	}
+                fuse_resolve_fd_init (state, &state->resolve, state->fd);
+        } else {
+                fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        }
 
         /*
          * This is just stub code demonstrating how to retrieve
@@ -921,14 +1181,15 @@ fuse_setattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_setattr_resume);
 }
 
-
 static int
 fuse_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         fuse_state_t *state = frame->root->state;
         fuse_in_header_t *finh = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": %s() %s => 0", frame->root->unique,
@@ -953,19 +1214,17 @@ fuse_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 static int
 fuse_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+                struct iatt *postbuf, dict_t *xdata)
 {
-        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno);
+        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 }
 
-
 static int
 fuse_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         if (op_ret == -1 && op_errno == ENOTSUP)
                 GF_LOG_OCCASIONALLY (gf_fuse_xattr_enotsup_log,
@@ -973,14 +1232,13 @@ fuse_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                      "extended attribute not supported "
                                      "by the backend storage");
 
-        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno);
+        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 }
 
-
 static int
 fuse_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         fuse_state_t     *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -988,11 +1246,14 @@ fuse_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
-        if (op_ret == 0)
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() %s => "
+                     "gfid: %s", op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op], state->loc.path,
+                     state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
 
         if (op_ret == 0) {
+                inode_unlink (state->loc.inode, state->loc.parent,
+                              state->loc.name);
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": %s() %s => 0", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path);
@@ -1014,7 +1275,6 @@ fuse_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 void
 fuse_access_resume (fuse_state_t *state)
 {
@@ -1022,7 +1282,7 @@ fuse_access_resume (fuse_state_t *state)
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": ACCESS %"PRIu64" (%s) resolution failed",
                         state->finh->unique, state->finh->nodeid,
-			uuid_utoa (state->resolve.gfid));
+                        uuid_utoa (state->resolve.gfid));
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1034,10 +1294,9 @@ fuse_access_resume (fuse_state_t *state)
                 state->finh->nodeid, state->mask);
 
         FUSE_FOP (state, fuse_err_cbk, GF_FOP_ACCESS, access,
-                  &state->loc, state->mask);
+                  &state->loc, state->mask, state->xdata);
 }
 
-
 static void
 fuse_access (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1055,11 +1314,10 @@ fuse_access (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 static int
 fuse_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, const char *linkname,
-                   struct iatt *buf)
+                   struct iatt *buf, dict_t *xdata)
 {
         fuse_state_t     *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -1067,6 +1325,12 @@ fuse_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d %"PRIu64": %s() => %s"
+                     " linkname: %s, gfid: %s", op_ret, op_errno,
+                     frame->root->unique, gf_fop_list[frame->root->op],
+                     state->loc.gfid, linkname,
+                     uuid_utoa (state->loc.gfid));
+
         if (op_ret > 0) {
                 ((char *)linkname)[op_ret] = '\0';
 
@@ -1089,7 +1353,6 @@ fuse_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 void
 fuse_readlink_resume (fuse_state_t *state)
 {
@@ -1107,10 +1370,9 @@ fuse_readlink_resume (fuse_state_t *state)
                 state->loc.path, uuid_utoa (state->loc.inode->gfid));
 
         FUSE_FOP (state, fuse_readlink_cbk, GF_FOP_READLINK,
-                  readlink, &state->loc, 4096);
+                  readlink, &state->loc, 4096, state->xdata);
 }
 
-
 static void
 fuse_readlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1118,22 +1380,21 @@ fuse_readlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         fuse_resolve_and_resume (state, fuse_readlink_resume);
 
         return;
 }
 
-
 void
 fuse_mknod_resume (fuse_state_t *state)
 {
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"MKNOD %"PRId64"/%s (%s/%s) resolution failed",
+                        "MKNOD %"PRIu64"/%s (%s/%s) resolution failed",
                         state->finh->nodeid, state->resolve.bname,
-			uuid_utoa (state->resolve.gfid), state->resolve.bname);
+                        uuid_utoa (state->resolve.gfid), state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1147,7 +1408,7 @@ fuse_mknod_resume (fuse_state_t *state)
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
-		state->loc.inode = NULL;
+                state->loc.inode = NULL;
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
@@ -1157,10 +1418,10 @@ fuse_mknod_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_MKNOD,
-                  mknod, &state->loc, state->mode, state->rdev, state->dict);
+                  mknod, &state->loc, state->mode, state->rdev, state->umask,
+                  state->xdata);
 }
 
-
 static void
 fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1181,43 +1442,14 @@ fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         uuid_generate (state->gfid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         state->mode = fmi->mode;
         state->rdev = fmi->rdev;
 
         priv = this->private;
 #if FUSE_KERNEL_MINOR_VERSION >=12
-        if (priv->proto_minor >= 12)
-                state->mode &= ~fmi->umask;
-        if (priv->proto_minor >= 12 && priv->acl) {
-                state->dict = dict_new ();
-                if (!state->dict) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKNOD Failed to allocate a param dictionary");
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "umask", fmi->umask);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKNOD Failed adding umask to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "mode", fmi->mode);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKNOD Failed adding mode to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-        }
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKNOD");
 #endif
 
         fuse_resolve_and_resume (state, fuse_mknod_resume);
@@ -1225,15 +1457,14 @@ fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 void
 fuse_mkdir_resume (fuse_state_t *state)
 {
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"MKDIR %"PRId64" (%s/%s) resolution failed",
+                        "MKDIR %"PRIu64" (%s/%s) resolution failed",
                         state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname);
+                        state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1247,7 +1478,7 @@ fuse_mkdir_resume (fuse_state_t *state)
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
-		state->loc.inode = NULL;
+                state->loc.inode = NULL;
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
@@ -1257,10 +1488,9 @@ fuse_mkdir_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_MKDIR,
-                  mkdir, &state->loc, state->mode, state->dict);
+                  mkdir, &state->loc, state->mode, state->umask, state->xdata);
 }
 
-
 static void
 fuse_mkdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1275,42 +1505,13 @@ fuse_mkdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         uuid_generate (state->gfid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         state->mode = fmi->mode;
 
         priv = this->private;
 #if FUSE_KERNEL_MINOR_VERSION >=12
-        if (priv->proto_minor >= 12)
-                state->mode &= ~fmi->umask;
-        if (priv->proto_minor >= 12 && priv->acl) {
-                state->dict = dict_new ();
-                if (!state->dict) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKDIR Failed to allocate a param dictionary");
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "umask", fmi->umask);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKDIR Failed adding umask to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "mode", fmi->mode);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "MKDIR Failed adding mode to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-        }
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKDIR");
 #endif
 
         fuse_resolve_and_resume (state, fuse_mkdir_resume);
@@ -1318,15 +1519,14 @@ fuse_mkdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 void
 fuse_unlink_resume (fuse_state_t *state)
 {
         if (!state->loc.parent || !state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"UNLINK %"PRId64" (%s/%s) resolution failed",
-			state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname);
+                        "UNLINK %"PRIu64" (%s/%s) resolution failed",
+                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1337,10 +1537,9 @@ fuse_unlink_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_unlink_cbk, GF_FOP_UNLINK,
-                  unlink, &state->loc);
+                  unlink, &state->loc, 0, state->xdata);
 }
 
-
 static void
 fuse_unlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1349,7 +1548,7 @@ fuse_unlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_unlink_resume);
 
@@ -1361,9 +1560,9 @@ fuse_rmdir_resume (fuse_state_t *state)
 {
         if (!state->loc.parent || !state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"RMDIR %"PRId64" (%s/%s) resolution failed",
-			state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname);
+                        "RMDIR %"PRIu64" (%s/%s) resolution failed",
+                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1374,10 +1573,9 @@ fuse_rmdir_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_unlink_cbk, GF_FOP_RMDIR,
-                  rmdir, &state->loc, 0);
+                  rmdir, &state->loc, 0, state->xdata);
 }
 
-
 static void
 fuse_rmdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1386,22 +1584,21 @@ fuse_rmdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_rmdir_resume);
 
         return;
 }
 
-
 void
 fuse_symlink_resume (fuse_state_t *state)
 {
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"SYMLINK %"PRId64" (%s/%s) -> %s resolution failed",
+                        "SYMLINK %"PRIu64" (%s/%s) -> %s resolution failed",
                         state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname, state->name);
+                        state->resolve.bname, state->name);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1415,7 +1612,7 @@ fuse_symlink_resume (fuse_state_t *state)
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
-		state->loc.inode = NULL;
+                state->loc.inode = NULL;
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
@@ -1425,10 +1622,9 @@ fuse_symlink_resume (fuse_state_t *state)
                 state->loc.path, state->name);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_SYMLINK,
-                  symlink, state->name, &state->loc, state->dict);
+                  symlink, state->name, &state->loc, state->umask, state->xdata);
 }
 
-
 static void
 fuse_symlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1440,7 +1636,7 @@ fuse_symlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         uuid_generate (state->gfid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         state->name = gf_strdup (linkname);
 
@@ -1449,12 +1645,12 @@ fuse_symlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 int
 fuse_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
                  struct iatt *preoldparent, struct iatt *postoldparent,
-                 struct iatt *prenewparent, struct iatt *postnewparent)
+                 struct iatt *prenewparent, struct iatt *postnewparent,
+                 dict_t *xdata)
 {
         fuse_state_t     *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -1462,9 +1658,18 @@ fuse_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() "
+                     "path: %s parent: %s ==> path: %s parent: %s"
+                     "gfid: %s", op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op], state->loc.path,
+                     state->loc.parent?uuid_utoa (state->loc.parent->gfid):"",
+                     state->loc2.path,
+                     state->loc2.parent?uuid_utoa (state->loc2.parent->gfid):"",
+                     state->loc.inode?uuid_utoa (state->loc.inode->gfid):"");
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s -> %s => 0 (buf->ia_ino=%"PRId64")",
+                        "%"PRIu64": %s -> %s => 0 (buf->ia_ino=%"PRIu64")",
                         frame->root->unique, state->loc.path, state->loc2.path,
                         buf->ia_ino);
 
@@ -1504,11 +1709,11 @@ fuse_rename_resume (fuse_state_t *state)
         if (!state->loc.parent || !state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "RENAME %"PRIu64" %s/%s -> %s/%s src resolution failed",
-			state->finh->unique,
-			uuid_utoa_r (state->resolve.gfid, loc_uuid),
-			state->resolve.bname,
-			uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
-			state->resolve2.bname);
+                        state->finh->unique,
+                        uuid_utoa_r (state->resolve.gfid, loc_uuid),
+                        state->resolve.bname,
+                        uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
+                        state->resolve2.bname);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -1518,11 +1723,11 @@ fuse_rename_resume (fuse_state_t *state)
         if (!state->loc2.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "RENAME %"PRIu64" %s/%s -> %s/%s dst resolution failed",
-			state->finh->unique,
-			uuid_utoa_r (state->resolve.gfid, loc_uuid),
-			state->resolve.bname,
-			uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
-			state->resolve2.bname);
+                        state->finh->unique,
+                        uuid_utoa_r (state->resolve.gfid, loc_uuid),
+                        state->resolve.bname,
+                        uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
+                        state->resolve2.bname);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -1538,10 +1743,9 @@ fuse_rename_resume (fuse_state_t *state)
                 state->loc2.path, loc2_uuid);
 
         FUSE_FOP (state, fuse_rename_cbk, GF_FOP_RENAME,
-                  rename, &state->loc, &state->loc2);
+                  rename, &state->loc, &state->loc2, state->xdata);
 }
 
-
 static void
 fuse_rename (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1552,16 +1756,15 @@ fuse_rename (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, oldname);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, oldname);
 
-	fuse_resolve_entry_init (state, &state->resolve2, fri->newdir, newname);
+        fuse_resolve_entry_init (state, &state->resolve2, fri->newdir, newname);
 
         fuse_resolve_and_resume (state, fuse_rename_resume);
 
         return;
 }
 
-
 void
 fuse_link_resume (fuse_state_t *state)
 {
@@ -1577,11 +1780,11 @@ fuse_link_resume (fuse_state_t *state)
         state->resolve.op_ret = 0;
         state->resolve2.op_ret = 0;
 
-	if (state->loc.inode) {
-		inode_unref (state->loc.inode);
-		state->loc.inode = NULL;
-	}
-	state->loc.inode = inode_ref (state->loc2.inode);
+        if (state->loc.inode) {
+                inode_unref (state->loc.inode);
+                state->loc.inode = NULL;
+        }
+        state->loc.inode = inode_ref (state->loc2.inode);
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": LINK() %s -> %s",
@@ -1589,10 +1792,9 @@ fuse_link_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_LINK,
-                  link, &state->loc2, &state->loc);
+                  link, &state->loc2, &state->loc, state->xdata);
 }
 
-
 static void
 fuse_link (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1602,21 +1804,20 @@ fuse_link (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve2, fli->oldnodeid);
+        fuse_resolve_inode_init (state, &state->resolve2, fli->oldnodeid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_link_resume);
 
         return;
 }
 
-
 static int
 fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  fd_t *fd, inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         fuse_state_t           *state        = NULL;
         fuse_in_header_t       *finh         = NULL;
@@ -1632,6 +1833,8 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         finh     = state->finh;
         foo.open_flags = 0;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 foo.fh = (uintptr_t) fd;
 
@@ -1641,12 +1844,12 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         foo.open_flags |= FOPEN_DIRECT_IO;
 
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %p (ino=%"PRId64")",
+                        "%"PRIu64": %s() %s => %p (ino=%"PRIu64")",
                         frame->root->unique, gf_fop_list[frame->root->op],
                         state->loc.path, fd, buf->ia_ino);
 
                 buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &feo.attr);
+                gf_fuse_stat2attr (buf, &feo.attr, priv->enable_ino32);
 
                 linked_inode = inode_link (inode, state->loc.parent,
                                            state->loc.name, buf);
@@ -1684,6 +1887,7 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 #endif
                 iov_out[2].iov_base = &foo;
                 iov_out[2].iov_len = sizeof (foo);
+
                 if (send_fuse_iov (this, finh, iov_out, 3) == ENOENT) {
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "create(%s) got EINTR", state->loc.path);
@@ -1707,18 +1911,18 @@ out:
         return 0;
 }
 
-
 void
 fuse_create_resume (fuse_state_t *state)
 {
-        fd_t           *fd   = NULL;
-        fuse_private_t *priv = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                         "%"PRIu64" CREATE %s/%s resolution failed",
                         state->finh->unique, uuid_utoa (state->resolve.gfid),
-			state->resolve.bname);
+                        state->resolve.bname);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -1738,6 +1942,25 @@ fuse_create_resume (fuse_state_t *state)
         state->loc.inode = inode_new (state->loc.parent->table);
 
         fd = fd_create (state->loc.inode, state->finh->pid);
+        if (fd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE cannot create a new fd",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
 
         priv = state->this->private;
 
@@ -1752,11 +1975,10 @@ fuse_create_resume (fuse_state_t *state)
 
         FUSE_FOP (state, fuse_create_cbk, GF_FOP_CREATE,
                   create, &state->loc, state->flags, state->mode,
-                  fd, state->dict);
+                  state->umask, fd, state->xdata);
 
 }
 
-
 static void
 fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1781,56 +2003,26 @@ fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         uuid_generate (state->gfid);
 
-	fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
         state->mode = fci->mode;
         state->flags = fci->flags;
 
         priv = this->private;
 #if FUSE_KERNEL_MINOR_VERSION >=12
-        if (priv->proto_minor >= 12)
-                state->mode &= ~fci->umask;
-        if (priv->proto_minor >= 12 && priv->acl) {
-                state->dict = dict_new ();
-                if (!state->dict) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "CREATE Failed to allocate a param dictionary");
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "umask", fci->umask);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "CREATE Failed adding umask to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-                ret = dict_set_int16 (state->dict, "mode", fci->mode);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "CREATE Failed adding mode to request");
-                        dict_destroy (state->dict);
-                        send_fuse_err (this, finh, ENOMEM);
-                        free_fuse_state (state);
-                        return;
-                }
-        }
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fci, "CREATE");
 #endif
-
         fuse_resolve_and_resume (state, fuse_create_resume);
 
         return;
 }
 
-
 void
 fuse_open_resume (fuse_state_t *state)
 {
-        fd_t           *fd   = NULL;
-        fuse_private_t *priv = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
@@ -1851,6 +2043,17 @@ fuse_open_resume (fuse_state_t *state)
                 return;
         }
 
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPEN creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
         priv = state->this->private;
 
         state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
@@ -1862,10 +2065,9 @@ fuse_open_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_fd_cbk, GF_FOP_OPEN,
-                  open, &state->loc, state->flags, fd, 0);
+                  open, &state->loc, state->flags, fd, state->xdata);
 }
 
-
 static void
 fuse_open (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -1874,7 +2076,7 @@ fuse_open (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         state->flags = foi->flags;
 
@@ -1883,12 +2085,11 @@ fuse_open (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 static int
 fuse_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
                 struct iovec *vector, int32_t count,
-                struct iatt *stbuf, struct iobref *iobref)
+                struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
         fuse_state_t *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -1898,9 +2099,11 @@ fuse_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": READ => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRId64,
+                        "%"PRIu64": READ => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRIu64,
                         frame->root->unique,
                         op_ret, state->size, state->off, stbuf->ia_size);
 
@@ -1935,8 +2138,8 @@ fuse_readv_resume (fuse_state_t *state)
                 "%"PRIu64": READ (%p, size=%zu, offset=%"PRIu64")",
                 state->finh->unique, state->fd, state->size, state->off);
 
-        FUSE_FOP (state, fuse_readv_cbk, GF_FOP_READ,
-                  readv, state->fd, state->size, state->off, state->io_flags);
+        FUSE_FOP (state, fuse_readv_cbk, GF_FOP_READ, readv, state->fd,
+                  state->size, state->off, state->io_flags, state->xdata);
 }
 
 static void
@@ -1953,7 +2156,7 @@ fuse_readv (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fd = FH_TO_FD (fri->fh);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         /* See comment by similar code in fuse_settatr */
         priv = this->private;
@@ -1970,11 +2173,10 @@ fuse_readv (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_readv_resume);
 }
 
-
 static int
 fuse_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *stbuf, struct iatt *postbuf)
+                 struct iatt *stbuf, struct iatt *postbuf, dict_t *xdata)
 {
         fuse_state_t *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -1983,9 +2185,11 @@ fuse_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         finh = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": WRITE => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRId64,
+                        "%"PRIu64": WRITE => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRIu64,
                         frame->root->unique,
                         op_ret, state->size, state->off, stbuf->ia_size);
 
@@ -2027,11 +2231,12 @@ fuse_write_resume (fuse_state_t *state)
         iobref_add (iobref, iobuf);
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": WRITE (%p, size=%"PRId64", offset=%"PRId64")",
+                "%"PRIu64": WRITE (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
                 state->finh->unique, state->fd, state->size, state->off);
 
         FUSE_FOP (state, fuse_writev_cbk, GF_FOP_WRITE, writev, state->fd,
-                  &state->vector, 1, state->off, state->io_flags, iobref);
+                  &state->vector, 1, state->off, state->io_flags, iobref,
+                  state->xdata);
 
         iobref_unref (iobref);
 }
@@ -2064,7 +2269,7 @@ fuse_write (xlator_t *this, fuse_in_header_t *finh, void *msg)
         */
 
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         /* See comment by similar code in fuse_settatr */
         priv = this->private;
@@ -2081,15 +2286,13 @@ fuse_write (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 void
 fuse_flush_resume (fuse_state_t *state)
 {
         FUSE_FOP (state, fuse_err_cbk, GF_FOP_FLUSH,
-                  flush, state->fd);
+                  flush, state->fd, state->xdata);
 }
 
-
 static void
 fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2102,7 +2305,7 @@ fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fd = FH_TO_FD (ffi->fh);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         state->lk_owner = ffi->lock_owner;
 
@@ -2114,18 +2317,27 @@ fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
+int
+fuse_internal_release (xlator_t *this, fd_t *fd)
+{
+        //This is a place holder function to prevent "xlator does not implement
+        //release_cbk" Warning log.
+        //Actual release happens as part of fuse_release which gets executed
+        //when kernel fuse sends it.
+        return 0;
+}
 
 static void
 fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri    = msg;
-        fd_t                   *new_fd = NULL;
-        fd_t                   *fd     = NULL;
-        uint64_t                val    = 0;
-        int                     ret    = 0;
-        fuse_state_t           *state  = NULL;
-        fuse_fd_ctx_t          *fdctx  = NULL;
-        fuse_private_t         *priv   = NULL;
+        struct fuse_release_in *fri       = msg;
+        fd_t                   *activefd = NULL;
+        fd_t                   *fd        = NULL;
+        uint64_t                val       = 0;
+        int                     ret       = 0;
+        fuse_state_t           *state     = NULL;
+        fuse_fd_ctx_t          *fdctx     = NULL;
+        fuse_private_t         *priv      = NULL;
 
         GET_STATE (this, finh, state);
         fd = FH_TO_FD (fri->fh);
@@ -2133,6 +2345,9 @@ fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         priv = this->private;
 
+        fuse_log_eh (this, "RELEASE(): %"PRIu64":, fd: %p, gfid: %s",
+                     finh->unique, fd, uuid_utoa (fd->inode->gfid));
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RELEASE %p", finh->unique, state->fd);
 
@@ -2140,9 +2355,9 @@ fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
         if (!ret) {
                 fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
                 if (fdctx) {
-                        new_fd = fdctx->fd;
-                        if (new_fd) {
-                                fd_unref (new_fd);
+                        activefd = fdctx->activefd;
+                        if (activefd) {
+                                fd_unref (activefd);
                         }
 
                         GF_FREE (fdctx);
@@ -2160,20 +2375,18 @@ fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 void
 fuse_fsync_resume (fuse_state_t *state)
 {
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": FSYNC %p", state->finh->unique,
-		state->fd);
+                state->fd);
 
         /* fsync_flags: 1 means "datasync" (no defines for this) */
         FUSE_FOP (state, fuse_fsync_cbk, GF_FOP_FSYNC,
-                  fsync, state->fd, state->flags & 1);
+                  fsync, state->fd, (state->flags & 1), state->xdata);
 }
 
-
 static void
 fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2186,19 +2399,19 @@ fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fd = FH_TO_FD (fsi->fh);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         state->flags = fsi->fsync_flags;
         fuse_resolve_and_resume (state, fuse_fsync_resume);
         return;
 }
 
-
 void
 fuse_opendir_resume (fuse_state_t *state)
 {
-        fd_t           *fd   = NULL;
-        fuse_private_t *priv = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
         priv = state->this->private;
 
@@ -2212,6 +2425,25 @@ fuse_opendir_resume (fuse_state_t *state)
         }
 
         fd = fd_create (state->loc.inode, state->finh->pid);
+        if (fd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR fd creation failed",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+        }
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
         state->fd = fd_ref (fd);
         state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
 
@@ -2220,10 +2452,9 @@ fuse_opendir_resume (fuse_state_t *state)
                 state->loc.path);
 
         FUSE_FOP (state, fuse_fd_cbk, GF_FOP_OPENDIR,
-                  opendir, &state->loc, fd);
+                  opendir, &state->loc, fd, state->xdata);
 }
 
-
 static void
 fuse_opendir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2240,7 +2471,6 @@ fuse_opendir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_opendir_resume);
 }
 
-
 unsigned char
 d_type_from_stat (struct iatt *buf)
 {
@@ -2274,10 +2504,10 @@ d_type_from_stat (struct iatt *buf)
         return d_type;
 }
 
-
 static int
 fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
 {
         fuse_state_t *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -2285,9 +2515,13 @@ fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         char         *buf = NULL;
         gf_dirent_t  *entry = NULL;
         struct fuse_dirent *fde = NULL;
+        fuse_private_t *priv = NULL;
 
         state = frame->root->state;
         finh  = state->finh;
+        priv = state->this->private;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret < 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -2307,6 +2541,11 @@ fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                            strlen (entry->d_name));
         }
 
+	if (size <= 0) {
+		send_fuse_data (this, finh, 0, 0);
+		goto out;
+	}
+
         buf = GF_CALLOC (1, size, gf_fuse_mt_char);
         if (!buf) {
                 gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
@@ -2319,10 +2558,7 @@ fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         size = 0;
         list_for_each_entry (entry, &entries->list, list) {
                 fde = (struct fuse_dirent *)(buf + size);
-                fde->ino = entry->d_ino;
-                fde->off = entry->d_off;
-                fde->namelen = strlen (entry->d_name);
-                strncpy (fde->name, entry->d_name, fde->namelen);
+                gf_fuse_fill_dirent (entry, fde, priv->enable_ino32);
                 size += FUSE_DIRENT_SIZE (fde);
         }
 
@@ -2334,25 +2570,22 @@ fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 out:
         free_fuse_state (state);
         STACK_DESTROY (frame->root);
-        if (buf)
-                GF_FREE (buf);
+        GF_FREE (buf);
         return 0;
 
 }
 
-
 void
 fuse_readdir_resume (fuse_state_t *state)
 {
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": READDIR (%p, size=%zu, offset=%"PRId64")",
+                "%"PRIu64": READDIR (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
                 state->finh->unique, state->fd, state->size, state->off);
 
         FUSE_FOP (state, fuse_readdir_cbk, GF_FOP_READDIR,
-                  readdir, state->fd, state->size, state->off);
+                  readdir, state->fd, state->size, state->off, state->xdata);
 }
 
-
 static void
 fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -2367,28 +2600,209 @@ fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fd = FH_TO_FD (fri->fh);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         fuse_resolve_and_resume (state, fuse_readdir_resume);
 }
 
 
+static int
+fuse_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		   dict_t *xdata)
+{
+	fuse_state_t *state = NULL;
+	fuse_in_header_t *finh = NULL;
+	int           size = 0;
+	char         *buf = NULL;
+	gf_dirent_t  *entry = NULL;
+	struct fuse_direntplus *fde = NULL;
+	struct fuse_entry_out *feo = NULL;
+	fuse_private_t         *priv   = NULL;
+
+	state = frame->root->state;
+	finh  = state->finh;
+	priv = this->private;
+
+	if (op_ret < 0) {
+		gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+			"%"PRIu64": READDIRP => -1 (%s)", frame->root->unique,
+			strerror (op_errno));
+
+		send_fuse_err (this, finh, op_errno);
+		goto out;
+	}
+
+	gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": READDIRP => %d/%"GF_PRI_SIZET",%"PRId64,
+		frame->root->unique, op_ret, state->size, state->off);
+
+	list_for_each_entry (entry, &entries->list, list) {
+		size += FUSE_DIRENT_ALIGN (FUSE_NAME_OFFSET_DIRENTPLUS +
+					   strlen (entry->d_name));
+	}
+
+	if (size <= 0) {
+		send_fuse_data (this, finh, 0, 0);
+		goto out;
+	}
+
+	buf = GF_CALLOC (1, size, gf_fuse_mt_char);
+	if (!buf) {
+		gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+			"%"PRIu64": READDIRP => -1 (%s)", frame->root->unique,
+			strerror (ENOMEM));
+		send_fuse_err (this, finh, ENOMEM);
+		goto out;
+	}
+
+	size = 0;
+	list_for_each_entry (entry, &entries->list, list) {
+		inode_t *linked_inode;
+
+		fde = (struct fuse_direntplus *)(buf + size);
+		feo = &fde->entry_out;
+		fde->dirent.ino = entry->d_ino;
+		fde->dirent.off = entry->d_off;
+		fde->dirent.type = entry->d_type;
+		fde->dirent.namelen = strlen (entry->d_name);
+		strncpy (fde->dirent.name, entry->d_name, fde->dirent.namelen);
+		size += FUSE_DIRENTPLUS_SIZE (fde);
+
+		if (!entry->inode)
+			continue;
+
+		entry->d_stat.ia_blksize = this->ctx->page_size;
+		gf_fuse_stat2attr (&entry->d_stat, &feo->attr, priv->enable_ino32);
+
+		linked_inode = inode_link (entry->inode, state->fd->inode,
+					   entry->d_name, &entry->d_stat);
+		if (!linked_inode)
+			continue;
+
+		inode_lookup (linked_inode);
+
+		feo->nodeid = inode_to_fuse_nodeid (linked_inode);
+
+		fuse_inode_set_need_lookup (linked_inode, this);
+
+		inode_unref (linked_inode);
+
+		feo->entry_valid =
+			calc_timeout_sec (priv->entry_timeout);
+		feo->entry_valid_nsec =
+			calc_timeout_nsec (priv->entry_timeout);
+		feo->attr_valid =
+			calc_timeout_sec (priv->attribute_timeout);
+		feo->attr_valid_nsec =
+			calc_timeout_nsec (priv->attribute_timeout);
+	}
+
+	send_fuse_data (this, finh, buf, size);
+out:
+	free_fuse_state (state);
+	STACK_DESTROY (frame->root);
+	GF_FREE (buf);
+	return 0;
+
+}
+
+
+void
+fuse_readdirp_resume (fuse_state_t *state)
+{
+	gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": READDIRP (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
+		state->finh->unique, state->fd, state->size, state->off);
+
+	FUSE_FOP (state, fuse_readdirp_cbk, GF_FOP_READDIRP,
+		  readdirp, state->fd, state->size, state->off, state->xdata);
+}
+
+
+static void
+fuse_readdirp (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+	struct fuse_read_in *fri = msg;
+
+	fuse_state_t *state = NULL;
+	fd_t         *fd = NULL;
+
+	GET_STATE (this, finh, state);
+	state->size = fri->size;
+	state->off = fri->offset;
+	fd = FH_TO_FD (fri->fh);
+	state->fd = fd;
+
+	fuse_resolve_fd_init (state, &state->resolve, fd);
+
+	fuse_resolve_and_resume (state, fuse_readdirp_resume);
+}
+
+#ifdef FALLOC_FL_KEEP_SIZE
+static int
+fuse_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		   struct iatt *postbuf, dict_t *xdata)
+{
+	return fuse_err_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+}
+
+static void
+fuse_fallocate_resume(fuse_state_t *state)
+{
+	gf_log("glusterfs-fuse", GF_LOG_TRACE,
+	       "%"PRIu64": FALLOCATE (%p, flags=%d, size=%zu, offset=%"PRId64")",
+	       state->finh->unique, state->fd, state->flags, state->size,
+	       state->off);
+
+	if (state->flags & FALLOC_FL_PUNCH_HOLE)
+		FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_DISCARD, discard,
+			 state->fd, state->off, state->size, state->xdata);
+	else
+		FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_FALLOCATE, fallocate,
+			 state->fd, (state->flags & FALLOC_FL_KEEP_SIZE),
+			 state->off, state->size, state->xdata);
+}
+
+static void
+fuse_fallocate(xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+	struct fuse_fallocate_in *ffi = msg;
+	fuse_state_t *state = NULL;
+
+	GET_STATE(this, finh, state);
+	state->off = ffi->offset;
+	state->size = ffi->length;
+	state->flags = ffi->mode;
+	state->fd = FH_TO_FD(ffi->fh);
+
+	fuse_resolve_fd_init(state, &state->resolve, state->fd);
+	fuse_resolve_and_resume(state, fuse_fallocate_resume);
+}
+#endif /* FALLOC_FL_KEEP_SIZE */
+
+
 static void
 fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri    = msg;
-        fd_t                   *new_fd = NULL;
-        uint64_t                val    = 0;
-        int                     ret    = 0;
-        fuse_state_t           *state  = NULL;
-        fuse_fd_ctx_t          *fdctx  = NULL;
-        fuse_private_t         *priv   = NULL;
+        struct fuse_release_in *fri       = msg;
+        fd_t                   *activefd = NULL;
+        uint64_t                val       = 0;
+        int                     ret       = 0;
+        fuse_state_t           *state     = NULL;
+        fuse_fd_ctx_t          *fdctx     = NULL;
+        fuse_private_t         *priv      = NULL;
 
         GET_STATE (this, finh, state);
         state->fd = FH_TO_FD (fri->fh);
 
         priv = this->private;
 
+        fuse_log_eh (this, "RELEASEDIR (): %"PRIu64": fd: %p, gfid: %s",
+                     finh->unique, state->fd,
+                     uuid_utoa (state->fd->inode->gfid));
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RELEASEDIR %p", finh->unique, state->fd);
 
@@ -2397,9 +2811,9 @@ fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         if (!ret) {
                 fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
                 if (fdctx) {
-                        new_fd = fdctx->fd;
-                        if (new_fd) {
-                                fd_unref (new_fd);
+                        activefd = fdctx->activefd;
+                        if (activefd) {
+                                fd_unref (activefd);
                         }
 
                         GF_FREE (fdctx);
@@ -2423,7 +2837,7 @@ void
 fuse_fsyncdir_resume (fuse_state_t *state)
 {
         FUSE_FOP (state, fuse_err_cbk, GF_FOP_FSYNCDIR,
-                  fsyncdir, state->fd, state->flags & 1);
+                  fsyncdir, state->fd, (state->flags & 1), state->xdata);
 
 }
 
@@ -2440,7 +2854,7 @@ fuse_fsyncdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         GET_STATE (this, finh, state);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         state->flags = fsi->fsync_flags;
         fuse_resolve_and_resume (state, fuse_fsyncdir_resume);
@@ -2448,10 +2862,10 @@ fuse_fsyncdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 static int
 fuse_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                 dict_t *xdata)
 {
         fuse_state_t *state = NULL;
         fuse_in_header_t *finh = NULL;
@@ -2462,6 +2876,10 @@ fuse_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         priv  = this->private;
         finh  = state->finh;
 
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s()",
+                     op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op]);
+
         if (op_ret == 0) {
 #ifndef GF_DARWIN_HOST_OS
                 /* MacFUSE doesn't respect anyof these tweaks */
@@ -2501,7 +2919,6 @@ fuse_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-
 void
 fuse_statfs_resume (fuse_state_t *state)
 {
@@ -2519,7 +2936,7 @@ fuse_statfs_resume (fuse_state_t *state)
                 "%"PRIu64": STATFS", state->finh->unique);
 
         FUSE_FOP (state, fuse_statfs_cbk, GF_FOP_STATFS,
-                  statfs, &state->loc);
+                  statfs, &state->loc, state->xdata);
 }
 
 
@@ -2530,9 +2947,9 @@ fuse_statfs (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
-	fuse_resolve_and_resume (state, fuse_statfs_resume);
+        fuse_resolve_and_resume (state, fuse_statfs_resume);
 }
 
 
@@ -2544,7 +2961,7 @@ fuse_setxattr_resume (fuse_state_t *state)
                         "%"PRIu64": SETXATTR %s/%"PRIu64" (%s) "
                         "resolution failed",
                         state->finh->unique, uuid_utoa (state->resolve.gfid),
-			state->finh->nodeid, state->name);
+                        state->finh->nodeid, state->name);
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
                 return;
@@ -2560,14 +2977,16 @@ fuse_setxattr_resume (fuse_state_t *state)
                         state->fd, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_setxattr_cbk, GF_FOP_FSETXATTR,
-                          fsetxattr, state->fd, state->dict, state->flags);
+                          fsetxattr, state->fd, state->xattr, state->flags,
+                          state->xdata);
         } else {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": SETXATTR %s/%"PRIu64" (%s)", state->finh->unique,
                         state->loc.path, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_setxattr_cbk, GF_FOP_SETXATTR,
-                          setxattr, &state->loc, state->dict, state->flags);
+                          setxattr, &state->loc, state->xattr, state->flags,
+                          state->xdata);
         }
 }
 
@@ -2599,22 +3018,27 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
 #endif
 
+        if (fuse_ignore_xattr_set (priv, name)) {
+                (void) send_fuse_err (this, finh, 0);
+                return;
+        }
+
         if (!priv->acl) {
-                if ((strcmp (name, "system.posix_acl_access") == 0) ||
-                    (strcmp (name, "system.posix_acl_default") == 0)) {
+                if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
+                    (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
                         send_fuse_err (this, finh, EOPNOTSUPP);
                         GF_FREE (finh);
                         return;
                 }
         }
 
-#ifdef DISABLE_SELINUX
-        if (!strncmp (name, "security.", 9)) {
-                send_fuse_err (this, finh, EOPNOTSUPP);
-                GF_FREE (finh);
-                return;
+        if (!priv->selinux) {
+                if (strncmp (name, "security.", 9) == 0) {
+                        send_fuse_err (this, finh, EOPNOTSUPP);
+                        GF_FREE (finh);
+                        return;
+                }
         }
-#endif
 
         /* Check if the command is for changing the log
            level of process or specific xlator */
@@ -2629,12 +3053,12 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 gf_log ("fuse", GF_LOG_TRACE,
                         "got request to invalidate %"PRIu64, finh->nodeid);
                 send_fuse_err (this, finh, 0);
-                fuse_invalidate (this, finh->nodeid);
+                fuse_invalidate_entry (this, finh->nodeid);
                 GF_FREE (finh);
                 return;
         }
 
-        if (!strcmp (GFID_XATTR_KEY, name)) {
+        if (!strcmp (GFID_XATTR_KEY, name) || !strcmp (GF_XATTR_VOL_ID_KEY, name)) {
                 send_fuse_err (this, finh, EPERM);
                 GF_FREE (finh);
                 return;
@@ -2643,10 +3067,10 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         GET_STATE (this, finh, state);
         state->size = fsi->size;
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
-        state->dict = get_new_dict ();
-        if (!state->dict) {
+        state->xattr = get_new_dict ();
+        if (!state->xattr) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": SETXATTR dict allocation failed",
                         finh->unique);
@@ -2669,9 +3093,9 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 gf_log (THIS->name, GF_LOG_ERROR, "value size zero");
                 dict_value = NULL;
         }
-        dict_set (state->dict, newkey,
+        dict_set (state->xattr, newkey,
                   data_from_dynptr ((void *)dict_value, fsi->size));
-        dict_ref (state->dict);
+        dict_ref (state->xattr);
 
         state->flags = fsi->flags;
         state->name = newkey;
@@ -2705,10 +3129,28 @@ send_fuse_xattr (xlator_t *this, fuse_in_header_t *finh, const char *value,
         }
 }
 
+/* filter out xattrs that need not be visible on the
+ * mount point. this is _specifically_ for geo-rep
+ * as of now, to prevent Rsync from crying out loud
+ * when it tries to setxattr() for selinux xattrs
+ */
+static int
+fuse_filter_xattr(char *key)
+{
+        int need_filter = 0;
+        struct fuse_private *priv = THIS->private;
+
+        if ((priv->client_pid == GF_CLIENT_PID_GSYNCD)
+            && fnmatch ("*.selinux*", key, FNM_PERIOD) == 0)
+                need_filter = 1;
+
+        return need_filter;
+}
+
 
 static int
 fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *dict)
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
         int             need_to_free_dict = 0;
         char           *value = "";
@@ -2717,11 +3159,13 @@ fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         data_t         *value_data = NULL;
         int             ret = -1;
         int32_t         len = 0;
-        data_pair_t    *trav = NULL;
+        int32_t         len_next = 0;
 
         state = frame->root->state;
         finh  = state->finh;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret >= 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": %s() %s => %d", frame->root->unique,
@@ -2743,22 +3187,24 @@ fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         } /* if(value_data)...else */
                 } else {
                         /* if callback for listxattr */
-                        trav = dict->members_list;
-                        while (trav) {
-                                len += strlen (trav->key) + 1;
-                                trav = trav->next;
-                        } /* while(trav) */
+                        /* we need to invoke fuse_filter_xattr() twice. Once
+                         * while counting size and then while filling buffer
+                         */
+                        len = dict_keys_join (NULL, 0, dict, fuse_filter_xattr);
+                        if (len < 0)
+                                goto out;
+
                         value = alloca (len + 1);
                         if (!value)
                                 goto out;
-                        len = 0;
-                        trav = dict->members_list;
-                        while (trav) {
-                                strcpy (value + len, trav->key);
-                                value[len + strlen (trav->key)] = '\0';
-                                len += strlen (trav->key) + 1;
-                                trav = trav->next;
-                        } /* while(trav) */
+
+                        len_next = dict_keys_join (value, len, dict,
+                                                   fuse_filter_xattr);
+                        if (len_next != len)
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "sizes not equal %d != %d",
+                                        len, len_next);
+
                         send_fuse_xattr (this, finh, value, len, state->size);
                 } /* if(state->name)...else */
         } else {
@@ -2803,13 +3249,15 @@ out:
 void
 fuse_getxattr_resume (fuse_state_t *state)
 {
+        char *value  = NULL;
+
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                         "%"PRIu64": GETXATTR %s/%"PRIu64" (%s) "
-			"resolution failed",
+                        "resolution failed",
                         state->finh->unique,
-			uuid_utoa (state->resolve.gfid),
-			state->finh->nodeid, state->name);
+                        uuid_utoa (state->resolve.gfid),
+                        state->finh->nodeid, state->name);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -2820,20 +3268,60 @@ fuse_getxattr_resume (fuse_state_t *state)
         state->fd = fd_lookup (state->loc.inode, state->finh->pid);
 #endif /* GF_TEST_FFOP */
 
+        if (state->name &&
+            (strcmp (state->name, VIRTUAL_GFID_XATTR_KEY) == 0)) {
+                /* send glusterfs gfid in binary form */
+
+                value = GF_CALLOC (16 + 1, sizeof(char),
+                                   gf_common_mt_char);
+                if (!value) {
+                        send_fuse_err (state->this, state->finh, ENOMEM);
+                        goto internal_out;
+                }
+                memcpy (value, state->loc.inode->gfid, 16);
+
+                send_fuse_xattr (THIS, state->finh, value, 16, state->size);
+                GF_FREE (value);
+        internal_out:
+                free_fuse_state (state);
+                return;
+        }
+
+        if (state->name &&
+            (strcmp (state->name, VIRTUAL_GFID_XATTR_KEY_STR) == 0)) {
+                /* transform binary gfid to canonical form */
+
+                value = GF_CALLOC (UUID_CANONICAL_FORM_LEN + 1, sizeof(char),
+                                   gf_common_mt_char);
+                if (!value) {
+                        send_fuse_err (state->this, state->finh, ENOMEM);
+                        goto internal_out1;
+                }
+                uuid_utoa_r (state->loc.inode->gfid, value);
+
+                send_fuse_xattr (THIS, state->finh, value,
+                                 UUID_CANONICAL_FORM_LEN, state->size);
+                GF_FREE (value);
+        internal_out1:
+                free_fuse_state (state);
+                return;
+        }
+
+
         if (state->fd) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": GETXATTR %p/%"PRIu64" (%s)", state->finh->unique,
                         state->fd, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_FGETXATTR,
-                          fgetxattr, state->fd, state->name);
+                          fgetxattr, state->fd, state->name, state->xdata);
         } else {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": GETXATTR %s/%"PRIu64" (%s)", state->finh->unique,
                         state->loc.path, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
-                          getxattr, &state->loc, state->name);
+                          getxattr, &state->loc, state->name, state->xdata);
         }
 }
 
@@ -2841,15 +3329,16 @@ fuse_getxattr_resume (fuse_state_t *state)
 static void
 fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_getxattr_in *fgxi = msg;
-        char         *name = (char *)(fgxi + 1);
-
-        fuse_state_t *state = NULL;
-        struct fuse_private *priv = NULL;
-        int rv = 0;
-        char *newkey = NULL;
+        struct fuse_getxattr_in *fgxi     = msg;
+        char                    *name     = (char *)(fgxi + 1);
+        fuse_state_t            *state    = NULL;
+        struct fuse_private     *priv     = NULL;
+        int                      rv       = 0;
+        int                     op_errno  = EINVAL;
+        char                    *newkey   = NULL;
 
         priv = this->private;
+        GET_STATE (this, finh, state);
 
 #ifdef GF_DARWIN_HOST_OS
         if (fgxi->position) {
@@ -2865,45 +3354,43 @@ fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                         "%"PRIu64": GETXATTR %s/%"PRIu64" (%s):"
                         "refusing positioned getxattr",
                         finh->unique, state->loc.path, finh->nodeid, name);
-                send_fuse_err (this, finh, EINVAL);
-                FREE (finh);
-                return;
+                op_errno = EINVAL;
+                goto err;
         }
 #endif
 
         if (!priv->acl) {
-                if ((strcmp (name, "system.posix_acl_access") == 0) ||
-                    (strcmp (name, "system.posix_acl_default") == 0)) {
-                        send_fuse_err (this, finh, ENOTSUP);
-                        GF_FREE (finh);
-                        return;
+                if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
+                    (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
+                        op_errno = ENOTSUP;
+                        goto err;
                 }
         }
 
-#ifdef DISABLE_SELINUX
-        if (!strncmp (name, "security.", 9)) {
-                send_fuse_err (this, finh, ENODATA);
-                GF_FREE (finh);
-                return;
+        if (!priv->selinux) {
+                if (strncmp (name, "security.", 9) == 0) {
+                        op_errno = ENODATA;
+                        goto err;
+                }
         }
-#endif
-
-        GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         rv = fuse_flip_xattr_ns (priv, name, &newkey);
         if (rv) {
-                send_fuse_err (this, finh, ENOMEM);
-                free_fuse_state (state);
-                goto out;
+                op_errno = ENOMEM;
+                goto err;
         }
 
         state->size = fgxi->size;
         state->name = newkey;
 
         fuse_resolve_and_resume (state, fuse_getxattr_resume);
- out:
+
+        return;
+ err:
+        send_fuse_err (this, finh, op_errno);
+        free_fuse_state (state);
         return;
 }
 
@@ -2915,7 +3402,7 @@ fuse_listxattr_resume (fuse_state_t *state)
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                         "%"PRIu64": LISTXATTR %s/%"PRIu64
                         "resolution failed", state->finh->unique,
-			uuid_utoa (state->resolve.gfid), state->finh->nodeid);
+                        uuid_utoa (state->resolve.gfid), state->finh->nodeid);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -2932,14 +3419,14 @@ fuse_listxattr_resume (fuse_state_t *state)
                         state->fd, state->finh->nodeid);
 
                 FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_FGETXATTR,
-                          fgetxattr, state->fd, NULL);
+                          fgetxattr, state->fd, NULL, state->xdata);
         } else {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": LISTXATTR %s/%"PRIu64, state->finh->unique,
                         state->loc.path, state->finh->nodeid);
 
                 FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
-                          getxattr, &state->loc, NULL);
+                          getxattr, &state->loc, NULL, state->xdata);
         }
 }
 
@@ -2952,7 +3439,7 @@ fuse_listxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         state->size = fgxi->size;
 
@@ -2968,9 +3455,9 @@ fuse_removexattr_resume (fuse_state_t *state)
          if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                         "%"PRIu64": REMOVEXATTR %s/%"PRIu64" (%s) "
-			"resolution failed",
+                        "resolution failed",
                         state->finh->unique, uuid_utoa (state->resolve.gfid),
-			state->finh->nodeid, state->name);
+                        state->finh->nodeid, state->name);
 
                 send_fuse_err (state->this, state->finh, ENOENT);
                 free_fuse_state (state);
@@ -2987,14 +3474,14 @@ fuse_removexattr_resume (fuse_state_t *state)
                         state->fd, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_err_cbk, GF_FOP_FREMOVEXATTR,
-                          fremovexattr, state->fd, state->name);
+                          fremovexattr, state->fd, state->name, state->xdata);
         } else {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": REMOVEXATTR %s/%"PRIu64" (%s)", state->finh->unique,
                         state->loc.path, state->finh->nodeid, state->name);
 
                 FUSE_FOP (state, fuse_err_cbk, GF_FOP_REMOVEXATTR,
-                          removexattr, &state->loc, state->name);
+                          removexattr, &state->loc, state->name, state->xdata);
         }
 }
 
@@ -3009,7 +3496,7 @@ fuse_removexattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         int32_t       ret = -1;
         char *newkey = NULL;
 
-        if (!strcmp (GFID_XATTR_KEY, name)) {
+        if (!strcmp (GFID_XATTR_KEY, name) || !strcmp (GF_XATTR_VOL_ID_KEY, name)) {
                 send_fuse_err (this, finh, EPERM);
                 GF_FREE (finh);
                 return;
@@ -3019,7 +3506,7 @@ fuse_removexattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         GET_STATE (this, finh, state);
 
-	fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
         ret = fuse_flip_xattr_ns (priv, name, &newkey);
         if (ret) {
@@ -3039,13 +3526,16 @@ static int gf_fuse_lk_enosys_log;
 
 static int
 fuse_getlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
         fuse_state_t *state = NULL;
 
         state = frame->root->state;
         struct fuse_lk_out flo = {{0, }, };
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": ERR => 0", frame->root->unique);
@@ -3091,7 +3581,7 @@ fuse_getlk_resume (fuse_state_t *state)
                 "%"PRIu64": GETLK %p", state->finh->unique, state->fd);
 
         FUSE_FOP (state, fuse_getlk_cbk, GF_FOP_LK,
-                  lk, state->fd, F_GETLK, &state->lk_lock);
+                  lk, state->fd, F_GETLK, &state->lk_lock, state->xdata);
 }
 
 
@@ -3107,7 +3597,7 @@ fuse_getlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
         GET_STATE (this, finh, state);
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         convert_fuse_file_lock (&fli->lk, &state->lk_lock,
                                 fli->owner);
@@ -3122,7 +3612,8 @@ fuse_getlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
 static int
 fuse_setlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
         uint32_t      op    = 0;
         fuse_state_t *state = NULL;
@@ -3130,6 +3621,8 @@ fuse_setlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         state = frame->root->state;
         op    = state->finh->opcode;
 
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
         if (op_ret == 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                         "%"PRIu64": ERR => 0", frame->root->unique);
@@ -3151,11 +3644,10 @@ fuse_setlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "Returning EAGAIN Flock: "
                                 "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
-                                (unsigned long long) lock->l_start,
-                                (unsigned long long) lock->l_len,
-                                (unsigned long long) lock->l_pid,
+                                (unsigned long long) state->lk_lock.l_start,
+                                (unsigned long long) state->lk_lock.l_len,
+                                (unsigned long long) state->lk_lock.l_pid,
                                 lkowner_utoa (&frame->root->lk_owner));
-
                 } else  {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                                 "%"PRIu64": ERR => -1 (%s)",
@@ -3181,7 +3673,7 @@ fuse_setlk_resume (fuse_state_t *state)
 
         FUSE_FOP (state, fuse_setlk_cbk, GF_FOP_LK, lk, state->fd,
                   state->finh->opcode == FUSE_SETLK ? F_SETLK : F_SETLKW,
-                  &state->lk_lock);
+                  &state->lk_lock, state->xdata);
 }
 
 
@@ -3198,7 +3690,7 @@ fuse_setlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
         state->finh = finh;
         state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
         convert_fuse_file_lock (&fli->lk, &state->lk_lock,
                                 fli->owner);
@@ -3285,6 +3777,10 @@ fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fino.max_readahead = 1 << 17;
         fino.max_write = 1 << 17;
         fino.flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS;
+#if FUSE_KERNEL_MINOR_VERSION >= 17
+	if (fini->minor >= 17)
+		fino.flags |= FUSE_FLOCK_LOCKS;
+#endif
 #if FUSE_KERNEL_MINOR_VERSION >= 12
         if (fini->minor >= 12) {
             /* let fuse leave the umask processing to us, so that it does not
@@ -3313,8 +3809,8 @@ fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 }
                 priv->revchan_in  = pfd[0];
                 priv->revchan_out = pfd[1];
-                ret = pthread_create (&messenger, NULL, notify_kernel_loop,
-                                      this);
+                ret = gf_thread_create (&messenger, NULL, notify_kernel_loop,
+					this);
                 if (ret != 0) {
                         gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                                 "failed to start messenger daemon (%s)",
@@ -3324,15 +3820,70 @@ fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
                         goto out;
                 }
                 priv->reverse_fuse_thread_started = _gf_true;
+        } else {
+                /*
+                 * FUSE minor < 12 does not implement invalidate notifications.
+                 * This mechanism is required for fopen-keep-cache to operate
+                 * correctly. Disable and warn the user.
+                 */
+                if (priv->fopen_keep_cache) {
+                        gf_log("glusterfs-fuse", GF_LOG_WARNING, "FUSE version "
+                                "%d.%d does not support inval notifications. "
+                                "fopen-keep-cache disabled.", fini->major,
+                                fini->minor);
+                        priv->fopen_keep_cache = 0;
+                }
         }
+
         if (fini->minor >= 13) {
-                /* these values seemed to work fine during testing */
-                fino.max_background = 64;
-                fino.congestion_threshold = 48;
+                fino.max_background = priv->background_qlen;
+                fino.congestion_threshold = priv->congestion_threshold;
         }
         if (fini->minor < 9)
                 *priv->msg0_len_p = sizeof(*finh) + FUSE_COMPAT_WRITE_IN_SIZE;
 #endif
+        if (priv->use_readdirp) {
+                if (fini->flags & FUSE_DO_READDIRPLUS)
+                        fino.flags |= FUSE_DO_READDIRPLUS;
+        }
+
+	if (priv->fopen_keep_cache == 2) {
+		/* If user did not explicitly set --fopen-keep-cache[=off],
+		   then check if kernel support FUSE_AUTO_INVAL_DATA and ...
+		*/
+		if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+			/* ... enable fopen_keep_cache mode if supported.
+			*/
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "Detected "
+				"support for FUSE_AUTO_INVAL_DATA. Enabling "
+				"fopen_keep_cache automatically.");
+			fino.flags |= FUSE_AUTO_INVAL_DATA;
+			priv->fopen_keep_cache = 1;
+		} else {
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "No support "
+				"for FUSE_AUTO_INVAL_DATA. Disabling "
+				"fopen_keep_cache.");
+			/* ... else disable. */
+			priv->fopen_keep_cache = 0;
+		}
+	} else if (priv->fopen_keep_cache == 1) {
+		/* If user explicitly set --fopen-keep-cache[=on],
+		   then enable FUSE_AUTO_INVAL_DATA if possible.
+		*/
+		if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "fopen_keep_cache "
+				"is explicitly set. Enabling FUSE_AUTO_INVAL_DATA");
+			fino.flags |= FUSE_AUTO_INVAL_DATA;
+		} else {
+			gf_log ("glusterfs-fuse", GF_LOG_WARNING, "fopen_keep_cache "
+				"is explicitly set. Support for "
+				"FUSE_AUTO_INVAL_DATA is missing");
+		}
+	}
+
+	if (fini->flags & FUSE_ASYNC_DIO)
+		fino.flags |= FUSE_ASYNC_DIO;
+
         ret = send_fuse_obj (this, finh, &fino);
         if (ret == 0)
                 gf_log ("glusterfs-fuse", GF_LOG_INFO,
@@ -3472,14 +4023,17 @@ fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
         int          ret          = -1;
         dict_t      *xattr_req    = NULL;
         struct iatt  iatt         = {0, };
+        inode_t     *linked_inode = NULL;
 
         if ((loc == NULL) || (xl == NULL)) {
+                ret = -EINVAL;
                 goto out;
         }
 
         if (loc->inode == NULL) {
                 loc->inode = inode_new (xl->itable);
                 if (loc->inode == NULL) {
+                        ret = -ENOMEM;
                         goto out;
                 }
         }
@@ -3488,15 +4042,17 @@ fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
 
         xattr_req = dict_new ();
         if (xattr_req == NULL) {
+                ret = -ENOMEM;
                 goto out;
         }
 
         ret = syncop_lookup (xl, loc, xattr_req, &iatt, NULL, NULL);
-        if (ret < 0) {
+        if (ret < 0)
                 goto out;
-        }
 
-        inode_link (loc->inode, NULL, NULL, &iatt);
+        linked_inode = inode_link (loc->inode, NULL, NULL, &iatt);
+        inode_unref (loc->inode);
+        loc->inode = linked_inode;
 
         ret = 0;
 out:
@@ -3509,99 +4065,298 @@ out:
 
 
 int
-fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
-                 xlator_t *new_subvol)
+fuse_migrate_fd_open (xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                      xlator_t *old_subvol, xlator_t *new_subvol)
 {
-        int      ret                = -1;
-        loc_t    loc                = {0, };
-        char     create_in_progress = 0;
-        inode_t *old_inode          = NULL;
+        loc_t          loc       = {0, };
+        fd_t          *newfd     = NULL, *old_activefd = NULL;
+        fuse_fd_ctx_t *basefd_ctx = NULL;
+        fuse_fd_ctx_t *newfd_ctx = NULL;
+        int            ret       = 0, flags = 0;
 
-        /* could've used pthread_cond_wait, but that requires a cond variable to
-         * be mainted for each fd and that is a bit too much overhead.
-         */
-        do {
-                LOCK (&fd->inode->lock);
-                {
-                        if (uuid_is_null (fd->inode->gfid)) {
-                                create_in_progress = 1;
-                        } else {
-                                create_in_progress = 0;
-                        }
-                }
-                UNLOCK (&fd->inode->lock);
+        ret = inode_path (basefd->inode, NULL, (char **)&loc.path);
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "cannot construct path of gfid (%s) failed"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                        uuid_utoa (basefd->inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
 
-                if (create_in_progress) {
-                        gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                                "create call on fd (%p) is in progress, "
-                                "hence waiting", fd);
-                        sleep (1);
-                }
+        uuid_copy (loc.gfid, basefd->inode->gfid);
 
-        } while (create_in_progress);
+        loc.inode = inode_find (new_subvol->itable, basefd->inode->gfid);
 
-        if (fd->inode->table->xl == old_subvol) {
-                ret = syncop_fsync (old_subvol, fd);
+        if (loc.inode == NULL) {
+                ret = fuse_nameless_lookup (new_subvol, basefd->inode->gfid,
+                                            &loc);
                 if (ret < 0) {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "syncop_fsync failed (%s)", strerror (errno));
+                                "name-less lookup of gfid (%s) failed (%s)"
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                uuid_utoa (basefd->inode->gfid),
+                                strerror (-ret),
+                                old_subvol->name, old_subvol->graph->id,
+                                new_subvol->name, new_subvol->graph->id);
+                        ret = -1;
+                        goto out;
                 }
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "fd (%p) was not "
-                        "migrated during previous graph switch", fd);
+
+        }
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        newfd = fd_create (loc.inode, basefd->pid);
+        if (newfd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "cannot create new fd, hence not migrating basefd "
+                        "(ptr:%p inode-gfid:%s) "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (loc.inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                ret = -1;
+                goto out;
         }
 
-        loc.path = "";
-        loc.name = NULL;
+        newfd->flags = basefd->flags;
+	if (newfd->lk_ctx)
+		fd_lk_ctx_unref (newfd->lk_ctx);
+
+        newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
+
+        newfd_ctx = fuse_fd_ctx_check_n_create (this, newfd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", newfd_ctx, out);
+
+        if (IA_ISDIR (basefd->inode->ia_type)) {
+                ret = syncop_opendir (new_subvol, &loc, newfd);
+        } else {
+                flags = basefd->flags & ~(O_CREAT | O_EXCL | O_TRUNC);
+                ret = syncop_open (new_subvol, &loc, flags, newfd);
+        }
 
-        ret = fuse_nameless_lookup (new_subvol, fd->inode->gfid, &loc);
         if (ret < 0) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "name-less lookup of gfid (%s) failed (%s)",
-                        uuid_utoa (fd->inode->gfid), strerror (errno));
+                        "open on basefd (ptr:%p inode-gfid:%s) failed (%s)"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid), strerror (-ret),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                ret = -1;
                 goto out;
         }
 
-        old_inode = fd->inode;
+        fd_bind (newfd);
+
+        LOCK (&basefd->lock);
+        {
+                if (basefd_ctx->activefd != NULL) {
+                        old_activefd = basefd_ctx->activefd;
+                }
+
+                basefd_ctx->activefd = newfd;
+        }
+        UNLOCK (&basefd->lock);
+
+        if (old_activefd != NULL) {
+                fd_unref (old_activefd);
+        }
+
+        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                "migrated basefd (%p) to newfd (%p) (inode-gfid:%s)"
+                "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd, newfd,
+                uuid_utoa (basefd->inode->gfid),
+		old_subvol->name, old_subvol->graph->id,
+                new_subvol->name, new_subvol->graph->id);
+
+        ret = 0;
+
+out:
+        loc_wipe (&loc);
 
-        inode_ref (loc.inode);
+        return ret;
+}
 
-        LOCK (&fd->inode->lock);
+int
+fuse_migrate_locks (xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                    xlator_t *old_subvol, xlator_t *new_subvol)
+{
+        int	 ret      = -1;
+        dict_t	*lockinfo = NULL;
+        void    *ptr      = NULL;
+	fd_t    *newfd    = NULL;
+        fuse_fd_ctx_t *basefd_ctx         = NULL;
+
+
+	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
+		return 0;
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        LOCK (&basefd->lock);
         {
-                list_del_init (&fd->inode_list);
+                newfd = fd_ref (basefd_ctx->activefd);
         }
-        UNLOCK (&fd->inode->lock);
+        UNLOCK (&basefd->lock);
 
-        LOCK (&fd->lock);
+        ret = syncop_fgetxattr (old_subvol, oldfd, &lockinfo,
+                                GF_XATTR_LOCKINFO_KEY);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+			"getting lockinfo failed while migrating locks"
+			"(oldfd:%p newfd:%p inode-gfid:%s)"
+			"(old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_ptr (lockinfo, GF_XATTR_LOCKINFO_KEY, &ptr);
+        if (ptr == NULL) {
+                ret = 0;
+                gf_log (this->name, GF_LOG_INFO,
+                        "No lockinfo present on any of the bricks "
+			"(oldfd: %p newfd:%p inode-gfid:%s) "
+                        "(old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+
+                goto out;
+        }
+
+        ret = syncop_fsetxattr (new_subvol, newfd, lockinfo, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+			"migrating locks failed (oldfd:%p newfd:%p "
+			"inode-gfid:%s) (old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+                ret = -1;
+                goto out;
+        }
+
+out:
+	if (newfd)
+		fd_unref (newfd);
+
+        if (lockinfo != NULL) {
+                dict_unref (lockinfo);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_migrate_fd (xlator_t *this, fd_t *basefd, xlator_t *old_subvol,
+                 xlator_t *new_subvol)
+{
+        int            ret                = -1;
+        char           create_in_progress = 0;
+        fuse_fd_ctx_t *basefd_ctx         = NULL;
+        fd_t          *oldfd              = NULL;
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        LOCK (&basefd->lock);
         {
-                fd->inode = loc.inode;
+                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd
+                        : basefd;
+                fd_ref (oldfd);
         }
-        UNLOCK (&fd->lock);
+        UNLOCK (&basefd->lock);
 
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                ret = syncop_opendir (new_subvol, &loc, fd);
+        LOCK (&oldfd->inode->lock);
+        {
+                if (uuid_is_null (oldfd->inode->gfid)) {
+                        create_in_progress = 1;
+                } else {
+                        create_in_progress = 0;
+                }
+        }
+        UNLOCK (&oldfd->inode->lock);
+
+        if (create_in_progress) {
+                gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                        "create call on fd (%p) is in progress "
+                        "(basefd-ptr:%p basefd-inode.gfid:%s), "
+                        "hence deferring migration till application does an "
+                        "fd based operation on this fd"
+                        "(old-subvolume:%s-%d, new-subvolume:%s-%d)",
+                        oldfd, basefd, uuid_utoa (basefd->inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+
+                ret = 0;
+                goto out;
+        }
+
+        if (oldfd->inode->table->xl == old_subvol) {
+                ret = syncop_fsync (old_subvol, oldfd, 0);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "syncop_fsync failed (%s) on fd (%p)"
+                                "(basefd:%p basefd-inode.gfid:%s) "
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                strerror (-ret), oldfd, basefd,
+                                uuid_utoa (basefd->inode->gfid),
+                                old_subvol->name, old_subvol->graph->id,
+                                new_subvol->name, new_subvol->graph->id);
+                        ret = -1;
+                }
         } else {
-                ret = syncop_open (new_subvol, &loc, fd->flags, fd);
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "basefd (ptr:%p inode-gfid:%s) was not "
+                        "migrated during previous graph switch"
+                        "(old-subvolume:%s-%d new-subvolume: %s-%d)", basefd,
+                        basefd->inode->gfid,
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
         }
 
+        ret = fuse_migrate_fd_open (this, basefd, oldfd, old_subvol,
+                                    new_subvol);
         if (ret < 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "open on gfid (%s) failed (%s)",
-                        uuid_utoa (fd->inode->gfid), strerror (errno));
+                gf_log (this->name, GF_LOG_WARNING, "open corresponding to "
+                        "basefd (ptr:%p inode-gfid:%s) in new graph failed "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid), old_subvol->name,
+                        old_subvol->graph->id, new_subvol->name,
+                        new_subvol->graph->id);
                 goto out;
         }
 
-        fd_bind (fd);
+        ret = fuse_migrate_locks (this, basefd, oldfd, old_subvol,
+                                  new_subvol);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "migrating locks from old-subvolume (%s-%d) to "
+                        "new-subvolume (%s-%d) failed (inode-gfid:%s oldfd:%p "
+                        "basefd:%p)", old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id,
+                        uuid_utoa (basefd->inode->gfid), oldfd, basefd);
 
-        ret = 0;
+        }
 out:
-        if (loc.inode != NULL) {
-                inode_unref (loc.inode);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "migration of basefd "
+                        "(ptr:%p inode-gfid:%s) failed"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        oldfd ? uuid_utoa (oldfd->inode->gfid) : NULL,
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
         }
 
-        if (old_inode != NULL) {
-                inode_unref (old_inode);
-        }
+        fd_unref (oldfd);
 
         return ret;
 }
@@ -3617,6 +4372,8 @@ fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
         fdtable_t      *fdtable   = NULL;
         int             i         = 0;
         fd_t           *fd        = NULL;
+        int32_t         ret       = 0;
+        fuse_fd_ctx_t  *fdctx     = NULL;
 
         priv = this->private;
 
@@ -3626,12 +4383,32 @@ fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
         if (fdentries != NULL) {
                 for (i = 0; i < count; i++) {
                         fd = fdentries[i].fd;
-                        if (fd != NULL) {
-                                fuse_migrate_fd (this, fd, old_subvol,
-                                                 new_subvol);
+                        if (fd == NULL)
+                                continue;
+
+                        ret = fuse_migrate_fd (this, fd, old_subvol,
+                                               new_subvol);
+
+                        fdctx = fuse_fd_ctx_get (this, fd);
+                        if (fdctx) {
+                                LOCK (&fd->lock);
+                                {
+                                        if (ret < 0) {
+                                                fdctx->migration_failed = 1;
+                                        } else {
+                                                fdctx->migration_failed = 0;
+                                        }
+                                }
+                                UNLOCK (&fd->lock);
                         }
                 }
 
+                for (i = 0; i < count ; i++) {
+                        fd = fdentries[i].fd;
+                        if (fd)
+                                fd_unref (fd);
+                }
+
                 GF_FREE (fdentries);
         }
 
@@ -3665,12 +4442,6 @@ fuse_graph_switch_task (void *data)
         fuse_handle_blocked_locks (args->this, args->old_subvol,
                                    args->new_subvol);
 
-        pthread_mutex_lock (&args->lock);
-        {
-                args->complete = 1;
-                pthread_cond_broadcast (&args->cond);
-        }
-        pthread_mutex_unlock (&args->lock);
 out:
         return 0;
 }
@@ -3686,9 +4457,6 @@ fuse_graph_switch_args_alloc (void)
                 goto out;
         }
 
-        pthread_cond_init (&args->cond, NULL);
-        pthread_mutex_init (&args->lock, NULL);
-
 out:
         return args;
 }
@@ -3701,22 +4469,12 @@ fuse_graph_switch_args_destroy (fuse_graph_switch_args_t *args)
                 goto out;
         }
 
-        pthread_cond_destroy (&args->cond);
-        pthread_mutex_destroy (&args->lock);
-
         GF_FREE (args);
 out:
         return;
 }
 
 
-static int
-fuse_graph_switch_complete (int ret, call_frame_t *frame, void *data)
-{
-        return 0;
-}
-
-
 int
 fuse_handle_graph_switch (xlator_t *this, xlator_t *old_subvol,
                           xlator_t *new_subvol)
@@ -3739,27 +4497,14 @@ fuse_handle_graph_switch (xlator_t *this, xlator_t *old_subvol,
         args->old_subvol = old_subvol;
         args->new_subvol = new_subvol;
 
-        ret = synctask_new (this->ctx->env, fuse_graph_switch_task,
-                            fuse_graph_switch_complete, frame, args);
+        ret = synctask_new (this->ctx->env, fuse_graph_switch_task, NULL, frame,
+                            args);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "starting sync-task to "
                         "handle graph switch failed");
                 goto out;
         }
 
-        pthread_mutex_lock (&args->lock);
-        {
-                while (!args->complete) {
-                        ret = pthread_cond_wait (&args->cond, &args->lock);
-                        if (ret != 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cond_wait failed ret:%d errno:%d", ret,
-                                        errno);
-                        }
-                }
-        }
-        pthread_mutex_unlock (&args->lock);
-
         ret = 0;
 out:
         if (args != NULL) {
@@ -3799,11 +4544,11 @@ fuse_graph_sync (xlator_t *this)
                         ret = pthread_cond_wait (&priv->sync_cond,
                                                  &priv->sync_mutex);
                         if (ret != 0) {
-			          gf_log (this->name, GF_LOG_DEBUG,
-					  "timedwait returned non zero value "
-					  "ret: %d errno: %d", ret, errno);
-				  break;
-			}
+                                  gf_log (this->name, GF_LOG_DEBUG,
+                                          "timedwait returned non zero value "
+                                          "ret: %d errno: %d", ret, errno);
+                                  break;
+                        }
                 }
         }
 unlock:
@@ -3832,20 +4577,38 @@ unlock:
         return 0;
 }
 
+int
+fuse_get_mount_status (xlator_t *this)
+{
+        int             kid_status = -1;
+        fuse_private_t *priv = this->private;
+
+        if (read(priv->status_pipe[0],&kid_status, sizeof(kid_status)) < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "could not get mount status");
+                kid_status = -1;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "mount status is %d", kid_status);
+
+        close(priv->status_pipe[0]);
+        close(priv->status_pipe[1]);
+        return kid_status;
+}
 
 static void *
 fuse_thread_proc (void *data)
 {
-        char           *mount_point = NULL;
-        xlator_t       *this = NULL;
-        fuse_private_t *priv = NULL;
-        ssize_t         res = 0;
-        struct iobuf   *iobuf = NULL;
-        fuse_in_header_t *finh;
-        struct iovec iov_in[2];
-        void *msg = NULL;
-        const size_t msg0_size = sizeof (*finh) + 128;
-        fuse_handler_t **fuse_ops = NULL;
+        char                     *mount_point = NULL;
+        xlator_t                 *this = NULL;
+        fuse_private_t           *priv = NULL;
+        ssize_t                   res = 0;
+        struct iobuf             *iobuf = NULL;
+        fuse_in_header_t         *finh;
+        struct iovec              iov_in[2];
+        void                     *msg = NULL;
+        const size_t              msg0_size = sizeof (*finh) + 128;
+        fuse_handler_t          **fuse_ops = NULL;
+        struct pollfd             pfd[2] = {{0,}};
+        gf_boolean_t              mount_finished = _gf_false;
 
         this = data;
         priv = this->private;
@@ -3862,6 +4625,41 @@ fuse_thread_proc (void *data)
                 /* THIS has to be reset here */
                 THIS = this;
 
+                if (!mount_finished) {
+                        memset(pfd,0,sizeof(pfd));
+                        pfd[0].fd = priv->status_pipe[0];
+                        pfd[0].events = POLLIN | POLLHUP | POLLERR;
+                        pfd[1].fd = priv->fd;
+                        pfd[1].events = POLLIN | POLLHUP | POLLERR;
+                        if (poll(pfd,2,-1) < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "poll error %s", strerror(errno));
+                                break;
+                        }
+                        if (pfd[0].revents & POLLIN) {
+                                if (fuse_get_mount_status(this) != 0) {
+                                        break;
+                                }
+                                mount_finished = _gf_true;
+                        }
+                        else if (pfd[0].revents) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "mount pipe closed without status");
+                                break;
+                        }
+                        if (!pfd[1].revents) {
+                                continue;
+                        }
+                }
+
+                /*
+                 * We don't want to block on readv while we're still waiting
+                 * for mount status.  That means we only want to get here if
+                 * mount_status is true (meaning that our wait completed
+                 * already) or if we already called poll(2) on priv->fd to
+                 * make sure it's ready.
+                 */
+
                 if (priv->init_recvd)
                         fuse_graph_sync (this);
 
@@ -3901,13 +4699,20 @@ fuse_thread_proc (void *data)
                                         "terminating upon getting %s when "
                                         "reading /dev/fuse",
                                         errno == ENODEV ? "ENODEV" : "EBADF");
-
+                                fuse_log_eh (this, "glusterfs-fuse: terminating"
+                                             " upon getting %s when "
+                                             "reading /dev/fuse",
+                                             errno == ENODEV ? "ENODEV":
+                                             "EBADF");
                                 break;
                         }
                         if (errno != EINTR) {
                                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                                         "read from /dev/fuse returned -1 (%s)",
                                         strerror (errno));
+                                fuse_log_eh (this, "glusterfs-fuse: read from "
+                                             "/dev/fuse returned -1 (%s)",
+                                             strerror (errno));
                         }
 
                         goto cont_err;
@@ -3915,6 +4720,8 @@ fuse_thread_proc (void *data)
                 if (res < sizeof (finh)) {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                                 "short read on /dev/fuse");
+                        fuse_log_eh (this, "glusterfs-fuse: short read on "
+                                     "/dev/fuse");
                         break;
                 }
 
@@ -3933,6 +4740,8 @@ fuse_thread_proc (void *data)
                    ) {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
                                 "inconsistent read on /dev/fuse");
+                        fuse_log_eh (this, "glusterfs-fuse: inconsistent read "
+                                     "on /dev/fuse");
                         break;
                 }
 
@@ -3942,11 +4751,12 @@ fuse_thread_proc (void *data)
                         msg = iov_in[1].iov_base;
                 else {
                         if (res > msg0_size) {
-                                iov_in[0].iov_base =
-                                  GF_REALLOC (iov_in[0].iov_base, res);
-                                if (iov_in[0].iov_base)
+                                void *b = GF_REALLOC (iov_in[0].iov_base, res);
+                                if (b) {
+                                        iov_in[0].iov_base = b;
                                         finh = (fuse_in_header_t *)
                                                  iov_in[0].iov_base;
+                                }
                                 else {
                                         gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                                                 "Out of memory");
@@ -3967,13 +4777,11 @@ fuse_thread_proc (void *data)
                     finh->uid == priv->uid_map_root)
                         finh->uid = 0;
 
-#ifdef GF_DARWIN_HOST_OS
                 if (finh->opcode >= FUSE_OP_HIGH)
                         /* turn down MacFUSE specific messages */
                         fuse_enosys (this, finh, msg);
                 else
-#endif
-                fuse_ops[finh->opcode] (this, finh, msg);
+                        fuse_ops[finh->opcode] (this, finh, msg);
 
                 iobuf_unref (iobuf);
                 continue;
@@ -3983,8 +4791,11 @@ fuse_thread_proc (void *data)
                 GF_FREE (iov_in[0].iov_base);
         }
 
-        iobuf_unref (iobuf);
-        GF_FREE (iov_in[0].iov_base);
+        /*
+         * We could be in all sorts of states with respect to iobuf and iov_in
+         * by the time we get here, and it's just not worth untangling them if
+         * we're about to kill ourselves anyway.
+         */
 
         if (dict_get (this->options, ZR_MOUNTPOINT_OPT))
                 mount_point = data_to_str (dict_get (this->options,
@@ -3992,11 +4803,10 @@ fuse_thread_proc (void *data)
         if (mount_point) {
                 gf_log (this->name, GF_LOG_INFO,
                         "unmounting %s", mount_point);
-                dict_del (this->options, ZR_MOUNTPOINT_OPT);
         }
 
+        /* Kill the whole process, not just this thread. */
         kill (getpid(), SIGTERM);
-
         return NULL;
 }
 
@@ -4010,7 +4820,7 @@ fuse_itable_dump (xlator_t  *this)
         gf_proc_dump_add_section("xlator.mount.fuse.itable");
         inode_table_dump(this->itable, "xlator.mount.fuse.itable");
 
-	return 0;
+        return 0;
 }
 
 int32_t
@@ -4053,17 +4863,59 @@ fuse_priv_dump (xlator_t  *this)
                             (int)private->strict_volfile_check);
         gf_proc_dump_write("reverse_thread_started", "%d",
                            (int)private->reverse_fuse_thread_started);
+        gf_proc_dump_write("use_readdirp", "%d", private->use_readdirp);
 
         return 0;
 }
 
+int
+fuse_history_dump (xlator_t *this)
+{
+        int      ret    = -1;
+        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+
+        GF_VALIDATE_OR_GOTO ("fuse", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+
+        gf_proc_dump_build_key (key_prefix, "xlator.mount.fuse",
+                                "history");
+        gf_proc_dump_add_section (key_prefix);
+        eh_dump (this->history, NULL, dump_history_fuse);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+dump_history_fuse (circular_buffer_t *cb, void *data)
+{
+        char       *string   =  NULL;
+        struct tm  *tm       =  NULL;
+        char       timestr[256] = {0,};
+
+        string = (char *)cb->data;
+        tm = localtime (&cb->tv.tv_sec);
+
+        if (tm) {
+                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
+                gf_proc_dump_write ("TIME", "%s", timestr);
+        }
+
+        gf_proc_dump_write ("message", "%s\n", string);
+
+        return 0;
+}
 
 int
 fuse_graph_setup (xlator_t *this, glusterfs_graph_t *graph)
 {
-        inode_table_t  *itable = NULL;
-        int             ret = 0;
-        fuse_private_t *priv = NULL;
+        inode_table_t     *itable     = NULL;
+        int                ret        = 0, winds = 0;
+        fuse_private_t    *priv       = NULL;
+        glusterfs_graph_t *prev_graph = NULL;
 
         priv = this->private;
 
@@ -4084,13 +4936,30 @@ fuse_graph_setup (xlator_t *this, glusterfs_graph_t *graph)
 
         pthread_mutex_lock (&priv->sync_mutex);
         {
-                priv->next_graph = graph;
-                priv->event_recvd = 0;
+                prev_graph = priv->next_graph;
+
+                if ((prev_graph != NULL) && (prev_graph->id > graph->id)) {
+                        /* there was a race and an old graph was initialised
+                         * before new one.
+                         */
+                        prev_graph = graph;
+                } else {
+                        priv->next_graph = graph;
+                        priv->event_recvd = 0;
+
+                        pthread_cond_signal (&priv->sync_cond);
+                }
 
-                pthread_cond_signal (&priv->sync_cond);
+                if (prev_graph != NULL)
+                        winds = ((xlator_t *)prev_graph->top)->winds;
         }
         pthread_mutex_unlock (&priv->sync_mutex);
 
+        if ((prev_graph != NULL) && (winds == 0)) {
+                xlator_notify (prev_graph->top, GF_EVENT_PARENT_DOWN,
+                               prev_graph->top, NULL);
+        }
+
         gf_log ("fuse", GF_LOG_INFO, "switched to graph %d",
                 ((graph) ? graph->id : 0));
 
@@ -4141,8 +5010,8 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                 if (!private->fuse_thread_started) {
                         private->fuse_thread_started = 1;
 
-                        ret = pthread_create (&private->fuse_thread, NULL,
-                                              fuse_thread_proc, this);
+                        ret = gf_thread_create (&private->fuse_thread, NULL,
+						fuse_thread_proc, this);
                         if (ret != 0) {
                                 gf_log (this->name, GF_LOG_DEBUG,
                                         "pthread_create() failed (%s)",
@@ -4190,45 +5059,54 @@ mem_acct_init (xlator_t *this)
 
 
 static fuse_handler_t *fuse_std_ops[FUSE_OP_HIGH] = {
-        [FUSE_INIT]        = fuse_init,
-        [FUSE_DESTROY]     = fuse_destroy,
         [FUSE_LOOKUP]      = fuse_lookup,
         [FUSE_FORGET]      = fuse_forget,
         [FUSE_GETATTR]     = fuse_getattr,
         [FUSE_SETATTR]     = fuse_setattr,
-        [FUSE_OPENDIR]     = fuse_opendir,
-        [FUSE_READDIR]     = fuse_readdir,
-        [FUSE_RELEASEDIR]  = fuse_releasedir,
-        [FUSE_ACCESS]      = fuse_access,
         [FUSE_READLINK]    = fuse_readlink,
+        [FUSE_SYMLINK]     = fuse_symlink,
         [FUSE_MKNOD]       = fuse_mknod,
         [FUSE_MKDIR]       = fuse_mkdir,
         [FUSE_UNLINK]      = fuse_unlink,
         [FUSE_RMDIR]       = fuse_rmdir,
-        [FUSE_SYMLINK]     = fuse_symlink,
         [FUSE_RENAME]      = fuse_rename,
         [FUSE_LINK]        = fuse_link,
-        [FUSE_CREATE]      = fuse_create,
         [FUSE_OPEN]        = fuse_open,
         [FUSE_READ]        = fuse_readv,
         [FUSE_WRITE]       = fuse_write,
-        [FUSE_FLUSH]       = fuse_flush,
+        [FUSE_STATFS]      = fuse_statfs,
         [FUSE_RELEASE]     = fuse_release,
         [FUSE_FSYNC]       = fuse_fsync,
-        [FUSE_FSYNCDIR]    = fuse_fsyncdir,
-        [FUSE_STATFS]      = fuse_statfs,
         [FUSE_SETXATTR]    = fuse_setxattr,
         [FUSE_GETXATTR]    = fuse_getxattr,
         [FUSE_LISTXATTR]   = fuse_listxattr,
         [FUSE_REMOVEXATTR] = fuse_removexattr,
+        [FUSE_FLUSH]       = fuse_flush,
+        [FUSE_INIT]        = fuse_init,
+        [FUSE_OPENDIR]     = fuse_opendir,
+        [FUSE_READDIR]     = fuse_readdir,
+        [FUSE_RELEASEDIR]  = fuse_releasedir,
+        [FUSE_FSYNCDIR]    = fuse_fsyncdir,
         [FUSE_GETLK]       = fuse_getlk,
         [FUSE_SETLK]       = fuse_setlk,
         [FUSE_SETLKW]      = fuse_setlk,
+        [FUSE_ACCESS]      = fuse_access,
+        [FUSE_CREATE]      = fuse_create,
+     /* [FUSE_INTERRUPT] */
+     /* [FUSE_BMAP] */
+        [FUSE_DESTROY]     = fuse_destroy,
+     /* [FUSE_IOCTL] */
+     /* [FUSE_POLL] */
+     /* [FUSE_NOTIFY_REPLY] */
+	[FUSE_BATCH_FORGET]= fuse_batch_forget,
+#ifdef FALLOC_FL_KEEP_SIZE
+	[FUSE_FALLOCATE]   = fuse_fallocate,
+#endif /* FALLOC_FL_KEEP_SIZE */
+	[FUSE_READDIRPLUS] = fuse_readdirp,
 };
 
 
-static fuse_handler_t *fuse_dump_ops[FUSE_OP_HIGH] = {
-};
+static fuse_handler_t *fuse_dump_ops[FUSE_OP_HIGH];
 
 
 static void
@@ -4256,7 +5134,7 @@ fuse_dumper (xlator_t *this, fuse_in_header_t *finh, void *msg)
                         "failed to dump fuse message (R): %s",
                         strerror (errno));
 
-        return priv->fuse_ops0[finh->opcode] (this, finh, msg);
+        priv->fuse_ops0[finh->opcode] (this, finh, msg);
 }
 
 
@@ -4274,8 +5152,11 @@ init (xlator_t *this_xl)
         int                xl_name_allocated = 0;
         int                fsname_allocated = 0;
         glusterfs_ctx_t   *ctx = NULL;
-        gf_boolean_t       sync_mtab = _gf_false;
+        gf_boolean_t       sync_to_mount = _gf_false;
+        gf_boolean_t       fopen_keep_cache = _gf_false;
+        unsigned long      mntflags = 0;
         char              *mnt_args = NULL;
+        eh_t              *event = NULL;
 
         if (this_xl == NULL)
                 return -1;
@@ -4283,7 +5164,7 @@ init (xlator_t *this_xl)
         if (this_xl->options == NULL)
                 return -1;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = this_xl->ctx;
         if (!ctx)
                 return -1;
 
@@ -4354,25 +5235,21 @@ init (xlator_t *this_xl)
                 goto cleanup_exit;
         }
 
-        ret = dict_get_double (options, "attribute-timeout",
-                               &priv->attribute_timeout);
-        if (ret != 0)
-                priv->attribute_timeout = 1.0; /* default */
+        GF_OPTION_INIT ("attribute-timeout", priv->attribute_timeout, double,
+                        cleanup_exit);
 
-        ret = dict_get_double (options, "entry-timeout",
-                               &priv->entry_timeout);
-        if (ret != 0)
-                priv->entry_timeout = 1.0; /* default */
+        GF_OPTION_INIT ("entry-timeout", priv->entry_timeout, double,
+                        cleanup_exit);
 
-        ret = dict_get_int32 (options, "client-pid",
-                              &priv->client_pid);
-        if (ret == 0)
-                priv->client_pid_set = _gf_true;
+        GF_OPTION_INIT ("negative-timeout", priv->negative_timeout, double,
+                        cleanup_exit);
 
-        ret = dict_get_uint32 (options, "uid-map-root",
-                               &priv->uid_map_root);
-        if (ret != 0)
-                priv->uid_map_root = 0;
+        GF_OPTION_INIT ("client-pid", priv->client_pid, int32, cleanup_exit);
+        /* have to check & register the presence of client-pid manually */
+        priv->client_pid_set = !!dict_get (this_xl->options, "client-pid");
+
+        GF_OPTION_INIT ("uid-map-root", priv->uid_map_root, uint32,
+                        cleanup_exit);
 
         priv->direct_io_mode = 2;
         ret = dict_get_str (options, ZR_DIRECT_IO_OPT, &value_string);
@@ -4381,29 +5258,21 @@ init (xlator_t *this_xl)
                 GF_ASSERT (ret == 0);
         }
 
-        priv->strict_volfile_check = 0;
-        ret = dict_get_str (options, ZR_STRICT_VOLFILE_CHECK, &value_string);
-        if (ret == 0) {
-                ret = gf_string2boolean (value_string,
-                                         &priv->strict_volfile_check);
-                GF_ASSERT (ret == 0);
-        }
+        GF_OPTION_INIT (ZR_STRICT_VOLFILE_CHECK, priv->strict_volfile_check,
+                        bool, cleanup_exit);
+
+        GF_OPTION_INIT ("acl", priv->acl, bool, cleanup_exit);
 
-        priv->acl = 0;
-        ret = dict_get_str (options, "acl", &value_string);
-        if (ret == 0) {
-                ret = gf_string2boolean (value_string, &priv->acl);
-                GF_ASSERT (ret == 0);
-        }
         if (priv->uid_map_root)
                 priv->acl = 1;
 
-        priv->read_only = 0;
-        ret = dict_get_str (options, "read-only", &value_string);
-        if (ret == 0) {
-                ret = gf_string2boolean (value_string, &priv->read_only);
-                GF_ASSERT (ret == 0);
-        }
+        GF_OPTION_INIT ("selinux", priv->selinux, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("read-only", priv->read_only, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("enable-ino32", priv->enable_ino32, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("use-readdirp", priv->use_readdirp, bool, cleanup_exit);
 
         priv->fuse_dump_fd = -1;
         ret = dict_get_str (options, "dump-fuse", &value_string);
@@ -4422,14 +5291,70 @@ init (xlator_t *this_xl)
                 priv->fuse_dump_fd = ret;
         }
 
-        sync_mtab = _gf_false;
-        ret = dict_get_str (options, "sync-mtab", &value_string);
+        sync_to_mount = _gf_false;
+        ret = dict_get_str (options, "sync-to-mount", &value_string);
         if (ret == 0) {
                 ret = gf_string2boolean (value_string,
-                                         &sync_mtab);
+                                         &sync_to_mount);
                 GF_ASSERT (ret == 0);
         }
 
+	priv->fopen_keep_cache = 2;
+	if (dict_get (options, "fopen-keep-cache")) {
+		GF_OPTION_INIT("fopen-keep-cache", fopen_keep_cache, bool,
+			       cleanup_exit);
+		priv->fopen_keep_cache = fopen_keep_cache;
+	}
+
+        GF_OPTION_INIT("gid-timeout", priv->gid_cache_timeout, int32,
+                cleanup_exit);
+
+        GF_OPTION_INIT ("fuse-mountopts", priv->fuse_mountopts, str, cleanup_exit);
+
+        if (gid_cache_init(&priv->gid_cache, priv->gid_cache_timeout) < 0) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR, "Failed to initialize "
+                        "group cache.");
+                goto cleanup_exit;
+        }
+
+        /* default values seemed to work fine during testing */
+        GF_OPTION_INIT ("background-qlen", priv->background_qlen, int32,
+                        cleanup_exit);
+        GF_OPTION_INIT ("congestion-threshold", priv->congestion_threshold,
+                        int32, cleanup_exit);
+
+        GF_OPTION_INIT("no-root-squash", priv->no_root_squash, bool,
+                       cleanup_exit);
+        /* change the client_pid to no-root-squash pid only if the
+           client is none of defrag process, hadoop access and gsyncd process.
+        */
+        if (!priv->client_pid_set) {
+                if (priv->no_root_squash == _gf_true) {
+                        priv->client_pid_set = _gf_true;
+                        priv->client_pid = GF_CLIENT_PID_NO_ROOT_SQUASH;
+                }
+        }
+
+        /* user has set only background-qlen, not congestion-threshold,
+           use the fuse kernel driver formula to set congestion. ie, 75% */
+        if (dict_get (this_xl->options, "background-qlen") &&
+            !dict_get (this_xl->options, "congestion-threshold")) {
+                priv->congestion_threshold = (priv->background_qlen * 3) / 4;
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control as 75%% of "
+                        "background-queue length (ie, (.75 * %d) = %d",
+                        priv->background_qlen, priv->congestion_threshold);
+        }
+
+        /* congestion should not be higher than background queue length */
+        if (priv->congestion_threshold > priv->background_qlen) {
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control same as "
+                        "background-queue length (%d)",
+                        priv->background_qlen);
+                priv->congestion_threshold = priv->background_qlen;
+        }
+
         cmd_args = &this_xl->ctx->cmd_args;
         fsname = cmd_args->volfile;
         if (!fsname && cmd_args->volfile_server) {
@@ -4459,17 +5384,36 @@ init (xlator_t *this_xl)
                 goto cleanup_exit;
         }
 
-        gf_asprintf (&mnt_args, "%s%sallow_other,max_read=131072",
-                     priv->read_only ? "ro," : "",
-                     priv->acl ? "" : "default_permissions,");
+        if (priv->read_only)
+                mntflags |= MS_RDONLY;
+        gf_asprintf (&mnt_args, "%s%s%sallow_other,max_read=131072",
+                     priv->acl ? "" : "default_permissions,",
+                     priv->fuse_mountopts ? priv->fuse_mountopts : "",
+                     priv->fuse_mountopts ? "," : "");
         if (!mnt_args)
                 goto cleanup_exit;
 
-        priv->fd = gf_fuse_mount (priv->mount_point, fsname, mnt_args,
-                                  sync_mtab ? &ctx->mtab_pid : NULL);
+        if (pipe(priv->status_pipe) < 0) {
+                gf_log (this_xl->name, GF_LOG_ERROR,
+                        "could not create pipe to separate mount process");
+                goto cleanup_exit;
+        }
+
+        priv->fd = gf_fuse_mount (priv->mount_point, fsname, mntflags, mnt_args,
+                                  sync_to_mount ? &ctx->mnt_pid : NULL,
+                                  priv->status_pipe[1]);
         if (priv->fd == -1)
                 goto cleanup_exit;
 
+        event = eh_new (FUSE_EVENT_HISTORY_SIZE, _gf_false, NULL);
+        if (!event) {
+                gf_log (this_xl->name, GF_LOG_ERROR,
+                        "could not create a new event history");
+                goto cleanup_exit;
+        }
+
+        this_xl->history = event;
+
         pthread_mutex_init (&priv->fuse_dump_mutex, NULL);
         pthread_cond_init (&priv->sync_cond, NULL);
         pthread_mutex_init (&priv->sync_mutex, NULL);
@@ -4499,12 +5443,13 @@ cleanup_exit:
                 GF_FREE (fsname);
         if (priv) {
                 GF_FREE (priv->mount_point);
-                close (priv->fd);
-                close (priv->fuse_dump_fd);
+                if (priv->fd != -1)
+                        close (priv->fd);
+                if (priv->fuse_dump_fd != -1)
+                        close (priv->fuse_dump_fd);
                 GF_FREE (priv);
         }
-        if (mnt_args)
-                GF_FREE (mnt_args);
+        GF_FREE (mnt_args);
         return -1;
 }
 
@@ -4528,26 +5473,29 @@ fini (xlator_t *this_xl)
                 gf_log (this_xl->name, GF_LOG_INFO,
                         "Unmounting '%s'.", mount_point);
 
-                dict_del (this_xl->options, ZR_MOUNTPOINT_OPT);
                 gf_fuse_unmount (mount_point, priv->fd);
                 close (priv->fuse_dump_fd);
+                dict_del (this_xl->options, ZR_MOUNTPOINT_OPT);
         }
         /* Process should terminate once fuse xlator is finished.
          * Required for AUTH_FAILED event.
          */
-        raise (SIGTERM);
+        kill (getpid (), SIGTERM);
 }
 
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
 
 struct xlator_cbks cbks = {
+        .invalidate = fuse_invalidate,
+        .forget     = fuse_forget_cbk,
+        .release    = fuse_internal_release
 };
 
 
 struct xlator_dumpops dumpops = {
         .priv  = fuse_priv_dump,
         .inode = fuse_itable_dump,
+        .history = fuse_history_dump,
 };
 
 struct volume_options options[] = {
@@ -4561,13 +5509,20 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_PATH
         },
         { .key  = {ZR_ATTR_TIMEOUT_OPT},
-          .type = GF_OPTION_TYPE_DOUBLE
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "1.0"
         },
         { .key  = {ZR_ENTRY_TIMEOUT_OPT},
-          .type = GF_OPTION_TYPE_DOUBLE
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "1.0"
+        },
+        { .key  = {ZR_NEGATIVE_TIMEOUT_OPT},
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "0.0"
         },
         { .key  = {ZR_STRICT_VOLFILE_CHECK},
-          .type = GF_OPTION_TYPE_BOOL
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
         },
         { .key  = {"client-pid"},
           .type = GF_OPTION_TYPE_INT
@@ -4575,11 +5530,60 @@ struct volume_options options[] = {
         { .key  = {"uid-map-root"},
           .type = GF_OPTION_TYPE_INT
         },
-        { .key  = {"sync-mtab"},
+        { .key  = {"sync-to-mount"},
           .type = GF_OPTION_TYPE_BOOL
         },
         { .key = {"read-only"},
           .type = GF_OPTION_TYPE_BOOL
         },
+        { .key = {"fopen-keep-cache"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"gid-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "2"
+        },
+        { .key = {"acl"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"selinux"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"enable-ino32"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key  = {"background-qlen"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "64",
+          .min = 16,
+          .max = (64 * GF_UNIT_KB),
+        },
+        { .key  = {"congestion-threshold"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "48",
+          .min = 12,
+          .max = (64 * GF_UNIT_KB),
+        },
+        { .key = {"fuse-mountopts"},
+          .type = GF_OPTION_TYPE_STR
+        },
+        { .key = {"use-readdirp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "yes"
+        },
+        { .key = {"no-root-squash"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false",
+          .description = "This is the mount option for disabling the "
+          "root squash for the client irrespective of whether the root-squash "
+          "option for the volume is set or not. But this option is honoured "
+          "only for the trusted clients. For non trusted clients this value "
+          "does not have any affect and the volume option for root-squash is "
+          "honoured.",
+        },
         { .key = {NULL} },
 };
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index b7f0dfd50..f1c4cb3f0 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _GF_FUSE_BRIDGE_H_
 #define _GF_FUSE_BRIDGE_H_
 
@@ -55,9 +45,10 @@
 #include "list.h"
 #include "dict.h"
 #include "syncop.h"
+#include "gidcache.h"
 
 #if defined(GF_LINUX_HOST_OS) || defined(__NetBSD__)
-#define FUSE_OP_HIGH (FUSE_POLL + 1)
+#define FUSE_OP_HIGH (FUSE_READDIRPLUS + 1)
 #endif
 #ifdef GF_DARWIN_HOST_OS
 #define FUSE_OP_HIGH (FUSE_DESTROY + 1)
@@ -66,8 +57,6 @@
 
 #define MAX_FUSE_PROC_DELAY 1
 
-#define DISABLE_SELINUX 1
-
 typedef struct fuse_in_header fuse_in_header_t;
 typedef void (fuse_handler_t) (xlator_t *this, fuse_in_header_t *finh,
                                void *msg);
@@ -87,6 +76,7 @@ struct fuse_private {
         size_t              *msg0_len_p;
 
         double               entry_timeout;
+        double               negative_timeout;
         double               attribute_timeout;
 
         pthread_cond_t       sync_cond;
@@ -109,13 +99,37 @@ struct fuse_private {
         gf_boolean_t         client_pid_set;
         unsigned             uid_map_root;
         gf_boolean_t         acl;
+        gf_boolean_t         selinux;
         gf_boolean_t         read_only;
+	int32_t	             fopen_keep_cache;
+	int32_t		     gid_cache_timeout;
+        gf_boolean_t         enable_ino32;
+        /* This is the mount option for disabling the root-squash for the
+           mount irrespective of whether the root-squash option for the
+           volume is set or not. But this option is honoured only for
+           thr trusted clients. For non trusted clients this value does
+           not have any affect and the volume option for root-squash is
+           honoured.
+        */
+        gf_boolean_t        no_root_squash;
         fdtable_t           *fdtable;
+	gid_cache_t	     gid_cache;
+        char                *fuse_mountopts;
 
         /* For fuse-reverse-validation */
         int                  revchan_in;
         int                  revchan_out;
         gf_boolean_t         reverse_fuse_thread_started;
+
+        /* For communicating with separate mount thread. */
+        int                  status_pipe[2];
+
+        /* for fuse queue length and congestion threshold */
+        int background_qlen;
+        int congestion_threshold;
+
+        /* for using fuse-kernel readdirp*/
+        gf_boolean_t use_readdirp;
 };
 typedef struct fuse_private fuse_private_t;
 
@@ -123,9 +137,6 @@ struct fuse_graph_switch_args {
         xlator_t        *this;
         xlator_t        *old_subvol;
         xlator_t        *new_subvol;
-        pthread_cond_t   cond;
-        pthread_mutex_t  lock;
-        char             complete;
 };
 typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
 
@@ -134,6 +145,7 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
                              sizeof (struct fuse_notify_inval_entry_out) + \
                              NAME_MAX + 1))
 
+#define FUSE_EVENT_HISTORY_SIZE 1024
 
 #define _FH_TO_FD(fh) ((fd_t *)(uintptr_t)(fh))
 
@@ -141,9 +153,10 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
 
 #define FUSE_FOP(state, ret, op_num, fop, args ...)                     \
         do {                                                            \
-                call_frame_t *frame = NULL;                             \
-                xlator_t *xl = NULL;                                    \
-                int32_t   op_ret = 0, op_errno = 0;                     \
+                call_frame_t   *frame   = NULL;                         \
+                xlator_t       *xl      = NULL;                         \
+                int32_t         op_ret  = 0, op_errno = 0;              \
+                fuse_resolve_t *resolve = NULL;                         \
                                                                         \
                 frame = get_call_frame_for_req (state);                 \
                 if (!frame) {                                           \
@@ -170,32 +183,66 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
                 frame->root->op    = op_num;                            \
                 frame->op          = op_num;                            \
                                                                         \
+                if ( state->resolve_now ) {                             \
+                        resolve = state->resolve_now;                   \
+                } else {                                                \
+                        resolve = &(state->resolve);                    \
+                }                                                       \
+                                                                        \
                 xl = state->active_subvol;				\
                 if (!xl) {                                              \
                         gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \
-                                          "xl is NULL");                  \
+                                          "xl is NULL");                \
                         op_errno = ENOENT;                              \
                         op_ret = -1;                                    \
-                } else if (state->resolve.op_ret < 0) {                 \
-                        op_errno = state->resolve.op_errno;               \
-                        op_ret = -1;                                      \
-/*                        gf_log_callingfn ("glusterfs-fuse", GF_LOG_WARNING, \
-                                          "resolve failed (%s)",             \
-                                          strerror (op_errno));             */ \
-                } else if (state->resolve2.op_ret < 0) {                     \
-                        op_errno = state->resolve2.op_errno;                 \
-                        op_ret = -1;                                         \
-                        /* gf_log_callingfn ("glusterfs-fuse", GF_LOG_WARNING, \
-                                          "resolve of second entity "        \
-                                          "failed (%s)",                     \
-                                          strerror (op_errno));             */ \
-                }                                                            \
-                                                                             \
-                if (op_ret < 0) {                                            \
-                        send_fuse_err (state->this, state->finh, op_errno);  \
+                } else if (resolve->op_ret < 0) {                       \
+                        op_errno = resolve->op_errno;                   \
+                        op_ret = -1;                                    \
+                        if (op_num == GF_FOP_LOOKUP) {                  \
+                                gf_log ("glusterfs-fuse",               \
+                                        (op_errno == ENOENT ? GF_LOG_TRACE \
+                                         : GF_LOG_WARNING),             \
+                                        "%"PRIu64": %s() %s => -1 (%s)", \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        resolve->resolve_loc.path,      \
+                                        strerror (op_errno));           \
+                        } else {                                        \
+                                gf_log ("glusterfs-fuse",               \
+                                        GF_LOG_WARNING,                 \
+                                        "%"PRIu64": %s() inode "        \
+                                        "migration of %s failed (%s)",  \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        resolve->resolve_loc.path,      \
+                                        strerror (op_errno));           \
+                        }                                               \
+                } else if (state->resolve2.op_ret < 0) {                \
+                        op_errno = state->resolve2.op_errno;            \
+                        op_ret = -1;                                    \
+                        gf_log ("glusterfs-fuse",                       \
+                                GF_LOG_WARNING,                         \
+                                "%"PRIu64": %s() inode "                \
+                                "migration of %s failed (%s)",          \
+                                frame->root->unique,                    \
+                                gf_fop_list[frame->root->op],           \
+                                state->resolve2.resolve_loc.path,       \
+                                strerror (op_errno));                   \
+                }                                                       \
+                                                                        \
+                if (op_ret < 0) {                                       \
+                        send_fuse_err (state->this, state->finh, op_errno); \
                         free_fuse_state (state);                        \
                         STACK_DESTROY (frame->root);                    \
                 } else {                                                \
+                        if (state->this->history)                       \
+                                gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \
+                                           "(%s)", frame->root->unique, \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           (state->fd == NULL)?         \
+                                           uuid_utoa (state->loc.gfid): \
+                                           uuid_utoa (state->fd->inode->gfid));\
                         STACK_WIND (frame, ret, xl, xl->fops->fop, args); \
                 }                                                       \
                                                                         \
@@ -204,7 +251,9 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
 
 #define FUSE_FOP_COOKIE(state, xl, ret, cky, op_num, fop, args ...)     \
         do {                                                            \
-                call_frame_t *frame = NULL;                             \
+                call_frame_t *frame  = NULL;                            \
+                xlator_t     *xl     = NULL;                            \
+                int32_t       op_ret = 0, op_errno = 0;                 \
                                                                         \
                 frame = get_call_frame_for_req (state);                 \
                 if (!frame) {                                           \
@@ -222,7 +271,62 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
                 frame->root->state = state;                             \
                 frame->root->op    = op_num;                            \
 		frame->op          = op_num;				\
-                STACK_WIND_COOKIE (frame, ret, cky, xl, xl->fops->fop, args); \
+                                                                        \
+                xl = state->active_subvol;				\
+                if (!xl) {                                              \
+                        gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \
+                                          "xl is NULL");                \
+                        op_errno = ENOENT;                              \
+                        op_ret = -1;                                    \
+                } else if (state->resolve.op_ret < 0) {                 \
+                        op_errno = state->resolve.op_errno;             \
+                        op_ret = -1;                                    \
+                        if (op_num == GF_FOP_LOOKUP) {                  \
+                                gf_log ("glusterfs-fuse",               \
+                                        (op_errno == ENOENT ? GF_LOG_TRACE \
+                                         : GF_LOG_WARNING),             \
+                                        "%"PRIu64": %s() %s => -1 (%s)", \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        state->resolve.resolve_loc.path, \
+                                        strerror (op_errno));           \
+                        } else {                                        \
+                                gf_log ("glusterfs-fuse",               \
+                                        GF_LOG_WARNING,                 \
+                                        "%"PRIu64": %s() inode "        \
+                                        "migration of %s failed (%s)",  \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        state->resolve.resolve_loc.path, \
+                                        strerror (op_errno));           \
+                        }                                               \
+                } else if (state->resolve2.op_ret < 0) {                \
+                        op_errno = state->resolve2.op_errno;            \
+                        op_ret = -1;                                    \
+                        gf_log ("glusterfs-fuse",                       \
+                                GF_LOG_WARNING,                         \
+                                "%"PRIu64": %s() inode "                \
+                                "migration of %s failed (%s)",          \
+                                frame->root->unique,                    \
+                                gf_fop_list[frame->root->op],           \
+                                state->resolve2.resolve_loc.path,       \
+                                strerror (op_errno));                   \
+                }                                                       \
+                                                                        \
+                if (op_ret < 0) {                                       \
+                        send_fuse_err (state->this, state->finh, op_errno); \
+                        free_fuse_state (state);                        \
+                        STACK_DESTROY (frame->root);                    \
+                } else {                                                \
+                        if (xl->history)                                \
+                                gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \
+                                           "(%s)", frame->root->unique, \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           uuid_utoa (state->loc.gfid)); \
+                        STACK_WIND_COOKIE (frame, ret, cky, xl, xl->fops->fop, \
+                                           args);                       \
+                }                                                       \
         } while (0)
 
 #define GF_SELECT_LOG_LEVEL(_errno)                     \
@@ -246,7 +350,80 @@ typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
                 }                                                          \
         } while (0)
 
+#define FUSE_ENTRY_CREATE(this, priv, finh, state, fci, op)             \
+                do {                                                    \
+                        if (priv->proto_minor >= 12)                    \
+                                state->mode &= ~fci->umask;             \
+                        if (priv->proto_minor >= 12 && priv->acl) {     \
+                                state->xdata = dict_new ();             \
+                                if (!state->xdata) {                    \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s failed to allocate " \
+                                                "a param dictionary", op); \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                                state->umask = fci->umask;              \
+                                                                        \
+/* TODO: remove this after 3.4.0 release. keeping it for the            \
+   sake of backward compatibility with old (3.3.[01])                   \
+   releases till then. */                                               \
+                                ret = dict_set_int16 (state->xdata, "umask", \
+                                                      fci->umask);      \
+                                if (ret < 0) {                          \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s Failed adding umask"\
+                                                " to request", op);     \
+                                        dict_destroy (state->xdata);    \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                                ret = dict_set_int16 (state->xdata, "mode", \
+                                                      fci->mode);       \
+                                if (ret < 0) {                          \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s Failed adding mode " \
+                                                "to request", op);         \
+                                        dict_destroy (state->xdata);    \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                        }                                               \
+                } while (0)
+
+#define fuse_log_eh_fop(this, state, frame, op_ret, op_errno)               \
+        do {                                                            \
+                if (this->history) {                                    \
+                        if (state->fd)                                  \
+                                gf_log_eh ("op_ret: %d, op_errno: %d, " \
+                                           "%"PRIu64", %s () => %p, gfid: %s", \
+                                           op_ret, op_errno,            \
+                                           frame->root->unique,         \
+                                           gf_fop_list[frame->root->op], \
+                                           state->fd,                   \
+                                           uuid_utoa (state->fd->inode->gfid)); \
+                        else                                            \
+                                gf_log_eh ("op_ret: %d, op_errno: %d, " \
+                                           "%"PRIu64", %s () => %s, gfid: %s", \
+                                           op_ret, op_errno,            \
+                                           frame->root->unique,         \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           uuid_utoa (state->loc.gfid)); \
+                }                                                       \
+        } while(0)
 
+#define fuse_log_eh(this, args...)              \
+        do {                                    \
+                if (this->history)              \
+                        gf_log_eh(args);        \
+        } while (0)
 
 static inline xlator_t *
 fuse_active_subvol (xlator_t *fuse)
@@ -297,7 +474,8 @@ typedef struct {
         size_t            size;
         unsigned long     nlookup;
         fd_t             *fd;
-        dict_t           *dict;
+        dict_t           *xattr;
+        dict_t           *xdata;
         char             *name;
         char              is_revalidate;
         gf_boolean_t      truncate_needed;
@@ -318,6 +496,7 @@ typedef struct {
         int            mask;
         dev_t          rdev;
         mode_t         mode;
+        mode_t         umask;
         struct iatt    attr;
         struct gf_flock   lk_lock;
         struct iovec   vector;
@@ -327,9 +506,10 @@ typedef struct {
         int32_t        fd_no;
 } fuse_state_t;
 
-typedef struct fuse_fd_ctx {
+typedef struct {
         uint32_t  open_flags;
-        fd_t     *fd;
+        char      migration_failed;
+        fd_t     *activefd;
 } fuse_fd_ctx_t;
 
 typedef void (*fuse_resume_fn_t) (fuse_state_t *state);
@@ -340,16 +520,18 @@ fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
 call_frame_t *get_call_frame_for_req (fuse_state_t *state);
 fuse_state_t *get_fuse_state (xlator_t *this, fuse_in_header_t *finh);
 void free_fuse_state (fuse_state_t *state);
-void gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa);
+void gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa,
+                        gf_boolean_t enable_ino32);
+void gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde,
+                          gf_boolean_t enable_ino32);
 uint64_t inode_to_fuse_nodeid (inode_t *inode);
 xlator_t *fuse_active_subvol (xlator_t *fuse);
 inode_t *fuse_ino_to_inode (uint64_t ino, xlator_t *fuse);
 int send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error);
 int fuse_gfid_set (fuse_state_t *state);
 int fuse_flip_xattr_ns (struct fuse_private *priv, char *okey, char **nkey);
-int fuse_xattr_alloc_default (char *okey, char **nkey);
-fuse_fd_ctx_t * __fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this);
-fuse_fd_ctx_t * fuse_fd_ctx_check_n_create (fd_t *fd, xlator_t *this);
+fuse_fd_ctx_t * __fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
+fuse_fd_ctx_t * fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
 
 int fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn);
 int fuse_resolve_inode_init (fuse_state_t *state, fuse_resolve_t *resolve,
@@ -358,4 +540,6 @@ int fuse_resolve_entry_init (fuse_state_t *state, fuse_resolve_t *resolve,
 			     ino_t par, char *name);
 int fuse_resolve_fd_init (fuse_state_t *state, fuse_resolve_t *resolve,
 			  fd_t *fd);
+int fuse_ignore_xattr_set (fuse_private_t *priv, char *key);
+int dump_history_fuse (circular_buffer_t *cb, void *data);
 #endif /* _GF_FUSE_BRIDGE_H_ */
diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c
index 91a6cba53..2774bdaa8 100644
--- a/xlators/mount/fuse/src/fuse-helpers.c
+++ b/xlators/mount/fuse/src/fuse-helpers.c
@@ -1,21 +1,15 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
+#ifdef __NetBSD__
+#define _KMEMUSER
+#endif
 
 #include "fuse-bridge.h"
 #if defined(GF_SOLARIS_HOST_OS)
@@ -28,14 +22,11 @@
 static void
 fuse_resolve_wipe (fuse_resolve_t *resolve)
 {
-        if (resolve->path)
-                GF_FREE ((void *)resolve->path);
+        GF_FREE ((void *)resolve->path);
 
-        if (resolve->bname)
-                GF_FREE ((void *)resolve->bname);
+        GF_FREE ((void *)resolve->bname);
 
-        if (resolve->resolved)
-                GF_FREE ((void *)resolve->resolved);
+        GF_FREE ((void *)resolve->resolved);
 
         if (resolve->fd)
                 fd_unref (resolve->fd);
@@ -70,10 +61,13 @@ free_fuse_state (fuse_state_t *state)
 
         loc_wipe (&state->loc2);
 
-        if (state->dict) {
-                dict_unref (state->dict);
-                state->dict = (void *)0xaaaaeeee;
+        if (state->xdata) {
+                dict_unref (state->xdata);
+                state->xdata = (void *)0xaaaaeeee;
         }
+        if (state->xattr)
+                dict_unref (state->xattr);
+
         if (state->name) {
                 GF_FREE (state->name);
                 state->name = NULL;
@@ -145,6 +139,7 @@ get_fuse_state (xlator_t *this, fuse_in_header_t *finh)
 }
 
 
+#define FUSE_MAX_AUX_GROUPS 32 /* We can get only up to 32 aux groups from /proc */
 void
 frame_fill_groups (call_frame_t *frame)
 {
@@ -167,6 +162,9 @@ frame_fill_groups (call_frame_t *frame)
         if (!fp)
                 goto out;
 
+	if (call_stack_alloc_groups (frame->root, FUSE_MAX_AUX_GROUPS) != 0)
+		goto out;
+
         while ((ptr = fgets (line, sizeof line, fp))) {
                 if (strncmp (ptr, "Groups:", 7) != 0)
                         continue;
@@ -183,7 +181,7 @@ frame_fill_groups (call_frame_t *frame)
                         if (!endptr || *endptr)
                                 break;
                         frame->root->groups[idx++] = id;
-                        if (idx == GF_MAX_AUX_GROUPS)
+                        if (idx == FUSE_MAX_AUX_GROUPS)
                                 break;
                 }
 
@@ -199,6 +197,7 @@ out:
         prcred_t    *prcred = (prcred_t *) scratch;
         FILE        *fp = NULL;
         int          ret = 0;
+	int          ngrps;
 
         ret = snprintf (filename, sizeof filename,
                         "/proc/%d/cred", frame->root->pid);
@@ -207,8 +206,11 @@ out:
                 fp = fopen (filename, "r");
                 if (fp != NULL) {
                         if (fgets (scratch, sizeof scratch, fp) != NULL) {
-                                frame->root->ngrps = MIN(prcred->pr_ngroups,
-                                                         GF_REQUEST_MAXGROUPS);
+                                ngrps = MIN(prcred->pr_ngroups,
+					    GF_MAX_AUX_GROUPS);
+				if (call_stack_alloc_groups (frame->root,
+							     ngrps) != 0)
+					return;
                         }
                         fclose (fp);
                  }
@@ -233,7 +235,9 @@ out:
 
         if (sysctl(name, namelen, &kp, &kplen, NULL, 0) != 0)
                 return;
-        ngroups = MIN(kp.kp_eproc.e_ucred.cr_ngroups, GF_REQUEST_MAXGROUPS);
+        ngroups = MIN(kp.kp_eproc.e_ucred.cr_ngroups, GF_MAX_AUX_GROUPS);
+	if (call_stack_alloc_groups (frame->root, ngroups) != 0)
+		return;
         for (i = 0; i < ngroups; i++)
                 frame->root->groups[i] = kp.kp_eproc.e_ucred.cr_groups[i];
         frame->root->ngrps = ngroups;
@@ -242,6 +246,56 @@ out:
 #endif /* GF_LINUX_HOST_OS */
 }
 
+/*
+ * Get the groups for the PID associated with this frame. If enabled,
+ * use the gid cache to reduce group list collection.
+ */
+static void get_groups(fuse_private_t *priv, call_frame_t *frame)
+{
+	int i;
+	const gid_list_t *gl;
+	gid_list_t agl;
+
+        if (-1 == priv->gid_cache_timeout) {
+                frame->root->ngrps = 0;
+                return;
+        }
+
+	if (!priv->gid_cache_timeout) {
+		frame_fill_groups(frame);
+		return;
+	}
+
+	gl = gid_cache_lookup(&priv->gid_cache, frame->root->pid,
+			      frame->root->uid, frame->root->gid);
+	if (gl) {
+		if (call_stack_alloc_groups (frame->root, gl->gl_count) != 0)
+			return;
+		frame->root->ngrps = gl->gl_count;
+		for (i = 0; i < gl->gl_count; i++)
+			frame->root->groups[i] = gl->gl_list[i];
+		gid_cache_release(&priv->gid_cache, gl);
+		return;
+	}
+
+	frame_fill_groups (frame);
+
+	agl.gl_id = frame->root->pid;
+	agl.gl_uid = frame->root->uid;
+	agl.gl_gid = frame->root->gid;
+	agl.gl_count = frame->root->ngrps;
+	agl.gl_list = GF_CALLOC(frame->root->ngrps, sizeof(gid_t),
+			gf_fuse_mt_gids_t);
+	if (!agl.gl_list)
+		return;
+
+	for (i = 0; i < frame->root->ngrps; i++)
+		agl.gl_list[i] = frame->root->groups[i];
+
+	if (gid_cache_add(&priv->gid_cache, &agl) != 1)
+		GF_FREE(agl.gl_list);
+}
+
 call_frame_t *
 get_call_frame_for_req (fuse_state_t *state)
 {
@@ -269,7 +323,7 @@ get_call_frame_for_req (fuse_state_t *state)
                                           state->lk_owner);
         }
 
-        frame_fill_groups (frame);
+	get_groups(priv, frame);
 
         if (priv && priv->client_pid_set)
                 frame->root->pid = priv->client_pid;
@@ -301,7 +355,9 @@ fuse_ino_to_inode (uint64_t ino, xlator_t *fuse)
 uint64_t
 inode_to_fuse_nodeid (inode_t *inode)
 {
-        if (!inode || __is_root_gfid (inode->gfid))
+        if (!inode)
+		return 0;
+	if (__is_root_gfid (inode->gfid))
                 return 1;
 
         return (unsigned long) inode;
@@ -389,39 +445,63 @@ fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
         }
         ret = 0;
 fail:
+        /* this should not happen as inode_path returns -1 when buf is NULL
+           for sure */
+        if (path && !loc->path)
+                GF_FREE (path);
         return ret;
 }
 
+/* Use the same logic as the Linux NFS-client */
+#define GF_FUSE_SQUASH_INO(ino) ((uint32_t) ino) ^ (ino >> 32)
 
 /* courtesy of folly */
 void
-gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa)
+gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa, gf_boolean_t enable_ino32)
 {
-        fa->ino        = st->ia_ino;
-        fa->size       = st->ia_size;
-        fa->blocks     = st->ia_blocks;
-        fa->atime      = st->ia_atime;
-        fa->mtime      = st->ia_mtime;
-        fa->ctime      = st->ia_ctime;
-        fa->atimensec  = st->ia_atime_nsec;
-        fa->mtimensec  = st->ia_mtime_nsec;
-        fa->ctimensec  = st->ia_ctime_nsec;
-        fa->mode       = st_mode_from_ia (st->ia_prot, st->ia_type);
-        fa->nlink      = st->ia_nlink;
-        fa->uid        = st->ia_uid;
-        fa->gid        = st->ia_gid;
-        fa->rdev       = makedev (ia_major (st->ia_rdev),
-                                  ia_minor (st->ia_rdev));
+        if (enable_ino32)
+                fa->ino = GF_FUSE_SQUASH_INO(st->ia_ino);
+        else
+                fa->ino = st->ia_ino;
+
+        fa->size        = st->ia_size;
+        fa->blocks      = st->ia_blocks;
+        fa->atime       = st->ia_atime;
+        fa->mtime       = st->ia_mtime;
+        fa->ctime       = st->ia_ctime;
+        fa->atimensec   = st->ia_atime_nsec;
+        fa->mtimensec   = st->ia_mtime_nsec;
+        fa->ctimensec   = st->ia_ctime_nsec;
+        fa->mode        = st_mode_from_ia (st->ia_prot, st->ia_type);
+        fa->nlink       = st->ia_nlink;
+        fa->uid         = st->ia_uid;
+        fa->gid         = st->ia_gid;
+        fa->rdev        = makedev (ia_major (st->ia_rdev),
+                                   ia_minor (st->ia_rdev));
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        fa->blksize    = st->ia_blksize;
+        fa->blksize     = st->ia_blksize;
 #endif
 #ifdef GF_DARWIN_HOST_OS
-        fa->crtime     = (uint64_t)-1;
-        fa->crtimensec = (uint32_t)-1;
-        fa->flags      = 0;
+        fa->crtime      = (uint64_t)-1;
+        fa->crtimensec  = (uint32_t)-1;
+        fa->flags       = 0;
 #endif
 }
 
+void
+gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde, gf_boolean_t enable_ino32)
+{
+        if (enable_ino32)
+                fde->ino = GF_FUSE_SQUASH_INO(entry->d_ino);
+        else
+                fde->ino = entry->d_ino;
+
+        fde->off         = entry->d_off;
+        fde->type        = entry->d_type;
+        fde->namelen     = strlen (entry->d_name);
+        strncpy (fde->name, entry->d_name, fde->namelen);
+}
+
 static int
 fuse_do_flip_xattr_ns (char *okey, const char *nns, char **nkey)
 {
@@ -447,7 +527,7 @@ fuse_do_flip_xattr_ns (char *okey, const char *nns, char **nkey)
         return ret;
 }
 
-int
+static int
 fuse_xattr_alloc_default (char *okey, char **nkey)
 {
         int ret = 0;
@@ -466,28 +546,12 @@ fuse_flip_xattr_ns (fuse_private_t *priv, char *okey, char **nkey)
 {
         int             ret       = 0;
         gf_boolean_t    need_flip = _gf_false;
-        gf_client_pid_t npid      = 0;
-
-        npid = priv->client_pid;
-        if (gf_client_pid_check (npid)) {
-                ret = fuse_xattr_alloc_default (okey, nkey);
-                goto out;
-        }
-
-        switch (npid) {
-                /*
-                 * These two cases will never execute as we check the
-                 * pid range above, but are kept to keep the compiler
-                 * happy.
-                 */
-        case GF_CLIENT_PID_MAX:
-        case GF_CLIENT_PID_MIN:
-                goto out;
 
+        switch (priv->client_pid) {
         case GF_CLIENT_PID_GSYNCD:
                 /* valid xattr(s): *xtime, volume-mark* */
                 gf_log("glusterfs-fuse", GF_LOG_DEBUG, "PID: %d, checking xattr(s): "
-                       "volume-mark*, *xtime", npid);
+                       "volume-mark*, *xtime", priv->client_pid);
                 if ( (strcmp (okey, UNPRIV_XA_NS".glusterfs.volume-mark") == 0)
                      || (fnmatch (UNPRIV_XA_NS".glusterfs.volume-mark.*", okey, FNM_PERIOD) == 0)
                      || (fnmatch (UNPRIV_XA_NS".glusterfs.*.xtime", okey, FNM_PERIOD) == 0) )
@@ -497,7 +561,7 @@ fuse_flip_xattr_ns (fuse_private_t *priv, char *okey, char **nkey)
         case GF_CLIENT_PID_HADOOP:
                 /* valid xattr(s): pathinfo */
                 gf_log("glusterfs-fuse", GF_LOG_DEBUG, "PID: %d, checking xattr(s): "
-                       "pathinfo", npid);
+                       "pathinfo", priv->client_pid);
                 if (strcmp (okey, UNPRIV_XA_NS".glusterfs.pathinfo") == 0)
                         need_flip = _gf_true;
                 break;
@@ -511,6 +575,36 @@ fuse_flip_xattr_ns (fuse_private_t *priv, char *okey, char **nkey)
                 /* if we cannot match, continue with what we got */
                 ret = fuse_xattr_alloc_default (okey, nkey);
         }
+
+        return ret;
+}
+
+int
+fuse_ignore_xattr_set (fuse_private_t *priv, char *key)
+{
+        int ret = 0;
+
+        /* don't mess with user namespace */
+        if (fnmatch ("user.*", key, FNM_PERIOD) == 0)
+                goto out;
+
+        if (priv->client_pid != GF_CLIENT_PID_GSYNCD)
+                goto out;
+
+        /* trusted NS check */
+        if (!((fnmatch ("*.glusterfs.*.xtime", key, FNM_PERIOD) == 0)
+              || (fnmatch ("*.glusterfs.volume-mark",
+                           key, FNM_PERIOD) == 0)
+              || (fnmatch ("*.glusterfs.volume-mark.*",
+                           key, FNM_PERIOD) == 0)
+              || (fnmatch ("glusterfs.gfid.newfile",
+                           key, FNM_PERIOD) == 0)))
+                ret = -1;
+
  out:
+        gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "%s setxattr: key [%s], "
+                " client pid [%d]", (ret ? "disallowing" : "allowing"), key,
+                priv->client_pid);
+
         return ret;
 }
diff --git a/xlators/mount/fuse/src/fuse-mem-types.h b/xlators/mount/fuse/src/fuse-mem-types.h
index 9c6a1c67a..28b4dfbdd 100644
--- a/xlators/mount/fuse/src/fuse-mem-types.h
+++ b/xlators/mount/fuse/src/fuse-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __FUSE_MEM_TYPES_H__
 #define __FUSE_MEM_TYPES_H__
 
@@ -31,6 +21,7 @@ enum gf_fuse_mem_types_ {
         gf_fuse_mt_fuse_state_t,
         gf_fuse_mt_fd_ctx_t,
         gf_fuse_mt_graph_switch_args_t,
+	gf_fuse_mt_gids_t,
         gf_fuse_mt_end
 };
 #endif
diff --git a/xlators/mount/fuse/src/fuse-resolve.c b/xlators/mount/fuse/src/fuse-resolve.c
index 5a09ea0d3..17d76d46b 100644
--- a/xlators/mount/fuse/src/fuse-resolve.c
+++ b/xlators/mount/fuse/src/fuse-resolve.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -30,7 +20,13 @@ fuse_resolve_all (fuse_state_t *state);
 int fuse_resolve_continue (fuse_state_t *state);
 int fuse_resolve_entry_simple (fuse_state_t *state);
 int fuse_resolve_inode_simple (fuse_state_t *state);
+int fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
+                     xlator_t *new_subvol);
 
+fuse_fd_ctx_t *
+fuse_fd_ctx_get (xlator_t *this, fd_t *fd);
+
+gf_boolean_t fuse_inode_needs_lookup (inode_t *inode, xlator_t *this);
 
 static int
 fuse_resolve_loc_touchup (fuse_state_t *state)
@@ -94,6 +90,7 @@ fuse_resolve_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                  resolve_loc->name, buf);
 
 	state->loc_now->inode = link_inode;
+
 out:
         loc_wipe (resolve_loc);
 
@@ -128,8 +125,8 @@ fuse_resolve_entry (fuse_state_t *state)
 
 int
 fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-                       dict_t *xattr, struct iatt *postparent)
+                       int op_ret, int op_errno, inode_t *inode,
+                       struct iatt *buf, dict_t *xattr, struct iatt *postparent)
 {
         fuse_state_t   *state      = NULL;
         fuse_resolve_t *resolve    = NULL;
@@ -149,7 +146,19 @@ fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         uuid_utoa (resolve->resolve_loc.gfid),
 			strerror (op_errno));
                 loc_wipe (&resolve->resolve_loc);
-                resolve->op_ret = -1;
+
+                /* resolve->op_ret can have 3 values: 0, -1, -2.
+                 * 0 : resolution was successful.
+                 * -1: parent inode could not be resolved.
+                 * -2: entry (inode corresponding to path) could not be resolved
+                 */
+
+                if (uuid_is_null (resolve->gfid)) {
+                        resolve->op_ret = -1;
+                } else {
+                        resolve->op_ret = -2;
+                }
+
                 resolve->op_errno = op_errno;
                 goto out;
         }
@@ -194,13 +203,16 @@ fuse_resolve_gfid (fuse_state_t *state)
                 uuid_copy (resolve_loc->gfid, resolve->gfid);
         }
 
-	resolve_loc->inode = inode_new (state->itable);
-	ret = inode_path (resolve_loc->inode, NULL,
-			  (char **)&resolve_loc->path);
+	/* inode may already exist in case we are looking up an inode which was
+	   linked through readdirplus */
+	resolve_loc->inode = inode_find (state->itable, resolve_loc->gfid);
+	if (!resolve_loc->inode)
+		resolve_loc->inode = inode_new (state->itable);
+	ret = loc_path (resolve_loc, NULL);
 
         if (ret <= 0) {
                 gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to get the path from inode %s",
+                        "failed to get the path for inode %s",
                         uuid_utoa (resolve->gfid));
         }
 
@@ -233,9 +245,26 @@ fuse_resolve_parent_simple (fuse_state_t *state)
 
 	parent = resolve->parhint;
 	if (parent->table == state->itable) {
+		if (fuse_inode_needs_lookup (parent, THIS))
+			return 1;
+
 		/* no graph switches since */
 		loc->parent = inode_ref (parent);
+		uuid_copy (loc->pargfid, parent->gfid);
 		loc->inode = inode_grep (state->itable, parent, loc->name);
+
+                /* nodeid for root is 1 and we blindly take the latest graph's
+                 * table->root as the parhint and because of this there is
+                 * ambiguity whether the entry should have existed or not, and
+                 * we took the conservative approach of assuming entry should
+                 * have been there even though it need not have (bug #804592).
+                 */
+                if ((loc->inode == NULL)
+                    && __is_root_gfid (parent->gfid)) {
+                        /* non decisive result - entry missing */
+                        return -1;
+                }
+
 		/* decisive result - resolution success */
 		return 0;
 	}
@@ -245,6 +274,10 @@ fuse_resolve_parent_simple (fuse_state_t *state)
 		/* non decisive result - parent missing */
 		return 1;
 	}
+	if (fuse_inode_needs_lookup (parent, THIS)) {
+		inode_unref (parent);
+		return 1;
+	}
 
 	loc->parent = parent;
         uuid_copy (loc->pargfid, resolve->pargfid);
@@ -294,15 +327,18 @@ fuse_resolve_inode_simple (fuse_state_t *state)
 	loc = state->loc_now;
 
 	inode = resolve->hint;
-	if (inode->table == state->itable) {
+	if (inode->table == state->itable)
 		inode_ref (inode);
-		goto found;
+	else
+		inode = inode_find (state->itable, resolve->gfid);
+
+        if (inode) {
+		if (!fuse_inode_needs_lookup (inode, THIS))
+			goto found;
+		/* inode was linked through readdirplus */
+		inode_unref (inode);
 	}
 
-        inode = inode_find (state->itable, resolve->gfid);
-        if (inode)
-		goto found;
-
         return 1;
 found:
 	loc->inode = inode;
@@ -327,27 +363,203 @@ fuse_resolve_inode (fuse_state_t *state)
         return 0;
 }
 
+
+int
+fuse_migrate_fd_task (void *data)
+{
+        int            ret        = -1;
+        fuse_state_t  *state      = NULL;
+        fd_t          *basefd     = NULL, *oldfd = NULL;
+        fuse_fd_ctx_t *basefd_ctx = NULL;
+        xlator_t      *old_subvol = NULL;
+
+        state = data;
+        if (state == NULL) {
+                goto out;
+        }
+
+        basefd = state->fd;
+
+        basefd_ctx = fuse_fd_ctx_get (state->this, basefd);
+        if (!basefd_ctx)
+                goto out;
+
+        LOCK (&basefd->lock);
+        {
+                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd : basefd;
+                fd_ref (oldfd);
+        }
+        UNLOCK (&basefd->lock);
+
+        old_subvol = oldfd->inode->table->xl;
+
+        ret = fuse_migrate_fd (state->this, basefd, old_subvol,
+                               state->active_subvol);
+
+        LOCK (&basefd->lock);
+        {
+                if (ret < 0) {
+                        basefd_ctx->migration_failed = 1;
+                } else {
+                        basefd_ctx->migration_failed = 0;
+                }
+        }
+        UNLOCK (&basefd->lock);
+
+        ret = 0;
+
+out:
+        if (oldfd)
+                fd_unref (oldfd);
+
+        return ret;
+}
+
+
+static inline int
+fuse_migrate_fd_error (xlator_t *this, fd_t *fd)
+{
+        fuse_fd_ctx_t *fdctx = NULL;
+        char           error = 0;
+
+        fdctx = fuse_fd_ctx_get (this, fd);
+        if (fdctx != NULL) {
+                if (fdctx->migration_failed) {
+                        error = 1;
+                }
+        }
+
+        return error;
+}
+
+#define FUSE_FD_GET_ACTIVE_FD(activefd, basefd)                 \
+        do {                                                    \
+                LOCK (&basefd->lock);                           \
+                {                                               \
+                        activefd = basefd_ctx->activefd ?       \
+                                basefd_ctx->activefd : basefd;  \
+                        if (activefd != basefd) {               \
+                                fd_ref (activefd);              \
+                        }                                       \
+                }                                               \
+                UNLOCK (&basefd->lock);                         \
+                                                                \
+                if (activefd == basefd) {                       \
+                        fd_ref (activefd);                      \
+                }                                               \
+        } while (0);
+
+
 static int
 fuse_resolve_fd (fuse_state_t *state)
 {
-        fuse_resolve_t  *resolve    = NULL;
-	fd_t            *fd         = NULL;
-	xlator_t        *active_subvol = NULL;
+        fuse_resolve_t *resolve            = NULL;
+	fd_t           *basefd             = NULL, *activefd = NULL;
+	xlator_t       *active_subvol      = NULL, *this = NULL;
+        int             ret                = 0;
+        char            fd_migration_error = 0;
+        fuse_fd_ctx_t  *basefd_ctx         = NULL;
 
         resolve = state->resolve_now;
 
-        fd = resolve->fd;
-	active_subvol = fd->inode->table->xl;
+        this = state->this;
+
+        basefd = resolve->fd;
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        if (basefd_ctx == NULL) {
+                gf_log (state->this->name, GF_LOG_WARNING,
+                        "fdctx is NULL for basefd (ptr:%p inode-gfid:%s), "
+                        "resolver erroring out with errno EINVAL",
+                        basefd, uuid_utoa (basefd->inode->gfid));
+                resolve->op_ret = -1;
+                resolve->op_errno = EINVAL;
+                goto resolve_continue;
+        }
+
+        FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
+
+        active_subvol = activefd->inode->table->xl;
 
-        if (state->active_subvol != active_subvol) {
+        fd_migration_error = fuse_migrate_fd_error (state->this, basefd);
+        if (fd_migration_error) {
                 resolve->op_ret = -1;
                 resolve->op_errno = EBADF;
-                gf_log ("fuse-resolve", GF_LOG_WARNING, "migration of fd (%p) "
-                        "did not complete, failing fop with EBADF", fd);
+        } else if (state->active_subvol != active_subvol) {
+                ret = synctask_new (state->this->ctx->env, fuse_migrate_fd_task,
+                                    NULL, NULL, state);
+
+                fd_migration_error = fuse_migrate_fd_error (state->this,
+                                                            basefd);
+                fd_unref (activefd);
+
+                FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
+                active_subvol = activefd->inode->table->xl;
+
+                if ((ret == -1) || fd_migration_error
+                    || (state->active_subvol != active_subvol)) {
+                        if (ret == -1) {
+                                gf_log (state->this->name, GF_LOG_WARNING,
+                                        "starting sync-task to migrate "
+                                        "basefd (ptr:%p inode-gfid:%s) failed "
+                                        "(old-subvolume:%s-%d "
+                                        "new-subvolume:%s-%d)",
+                                        basefd,
+                                        uuid_utoa (basefd->inode->gfid),
+                                        active_subvol->name,
+                                        active_subvol->graph->id,
+                                        state->active_subvol->name,
+                                        state->active_subvol->graph->id);
+                        } else {
+                                gf_log (state->this->name, GF_LOG_WARNING,
+                                        "fd migration of basefd "
+                                        "(ptr:%p inode-gfid:%s) failed "
+                                        "(old-subvolume:%s-%d "
+                                        "new-subvolume:%s-%d)",
+                                        basefd,
+                                        uuid_utoa (basefd->inode->gfid),
+                                        active_subvol->name,
+                                        active_subvol->graph->id,
+                                        state->active_subvol->name,
+                                        state->active_subvol->graph->id);
+                        }
+
+                        resolve->op_ret = -1;
+                        resolve->op_errno = EBADF;
+                } else {
+                        gf_log (state->this->name, GF_LOG_DEBUG,
+                                "basefd (ptr:%p inode-gfid:%s) migrated "
+                                "successfully in resolver "
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                basefd, uuid_utoa (basefd->inode->gfid),
+                                active_subvol->name, active_subvol->graph->id,
+                                state->active_subvol->name,
+                                state->active_subvol->graph->id);
+                }
+        }
+
+        if ((resolve->op_ret == -1) && (resolve->op_errno == EBADF)) {
+                gf_log ("fuse-resolve", GF_LOG_WARNING,
+                        "migration of basefd (ptr:%p inode-gfid:%s) "
+                        "did not complete, failing fop with EBADF "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid),
+                        active_subvol->name, active_subvol->graph->id,
+                        state->active_subvol->name,
+                        state->active_subvol->graph->id);
+        }
+
+        if (activefd != basefd) {
+                state->fd = fd_ref (activefd);
+                fd_unref (basefd);
         }
 
 	/* state->active_subvol = active_subvol; */
 
+resolve_continue:
+        if (activefd != NULL) {
+                fd_unref (activefd);
+        }
+
         fuse_resolve_continue (state);
 
         return 0;
@@ -362,15 +574,15 @@ fuse_gfid_set (fuse_state_t *state)
         if (uuid_is_null (state->gfid))
                 goto out;
 
-        if (!state->dict)
-                state->dict = dict_new ();
+        if (!state->xdata)
+                state->xdata = dict_new ();
 
-        if (!state->dict) {
+        if (!state->xdata) {
                 ret = -1;
                 goto out;
         }
 
-        ret = dict_set_static_bin (state->dict, "gfid-req",
+        ret = dict_set_static_bin (state->xdata, "gfid-req",
                                    state->gfid, sizeof (state->gfid));
 out:
         return ret;
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 9cdc69c1a..b7718b35f 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -1,20 +1,12 @@
 #!/bin/bash
-# (C) 2006, 2007, 2008 Gluster Inc. <http://www.gluster.com>
 #
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; either version 2 of
-# the License, or (at your option) any later version.
+# Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public
-# License along with this program; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-# Boston, MA 02110-1301 USA
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
 
 _init ()
 {
@@ -27,337 +19,611 @@ _init ()
     LOG_DEBUG=DEBUG;
     LOG_TRACE=TRACE;
 
+    HOST_NAME_MAX=64;
+
     prefix="@prefix@";
     exec_prefix=@exec_prefix@;
     cmd_line=$(echo "@sbindir@/glusterfs");
 
+    # check whether getfattr exists
+    export PATH
+    getfattr=$(which getfattr 2>/dev/null);
+    if [ $? -ne 0 ]; then
+        echo "WARNING: getfattr not found, certain checks will be skipped.."
+    fi
+
+    alias lsL='ls -L'
+    mounttab=/proc/mounts
+    uname_s=`uname -s`
+    case ${uname_s} in
+        NetBSD)
+            getinode="stat -f %i"
+            getdev="stat -f %d"
+            lgetinode="${getinode} -L"
+            lgetdev="${getdev} -L"
+            ;;
+        Linux)
+            getinode="stat -c %i"
+            getdev="stat -c %d"
+            lgetinode="${getinode} -L"
+            lgetdev="${getdev} -L"
+            ;;
+    esac
+
     UPDATEDBCONF=/etc/updatedb.conf
 }
 
+is_valid_hostname ()
+{
+    local server=$1
+
+    length=$(echo $server | wc -c)
+    if [ ${length} -gt ${HOST_NAME_MAX} ]; then
+        return 1
+    fi
+}
+
+parse_backup_volfile_servers ()
+{
+    local server_list=$1
+    local servers=""
+    local new_servers=""
+
+    servers=$(echo ${server_list} | sed 's/\:/ /g')
+    for server in ${servers}; do
+        is_valid_hostname ${server}
+        if [ $? -eq 1 ]; then
+            continue
+        fi
+        new_servers=$(echo "${new_servers} ${server}")
+    done
+
+    echo ${new_servers}
+}
+
+parse_volfile_servers ()
+{
+    local server_list=$1
+    local servers=""
+    local new_servers=""
+
+    servers=$(echo ${server_list} | sed 's/,/ /g')
+    for server in ${servers}; do
+        is_valid_hostname ${server}
+        if [ $? -eq 1 ]; then
+            continue
+        fi
+        new_servers=$(echo "${new_servers} ${server}")
+    done
+
+    echo ${new_servers}
+}
+
 start_glusterfs ()
 {
     if [ -n "$log_level_str" ]; then
-	case "$log_level_str" in
-	    "ERROR")
-		log_level=$LOG_ERROR;
-		;;
+        case "$( echo $log_level_str | awk '{print toupper($0)}')" in
+            "ERROR")
+                log_level=$LOG_ERROR;
+                ;;
             "INFO")
-                log_level=$LOG_INFO
+                log_level=$LOG_INFO;
                 ;;
-	    "DEBUG")
-		log_level=$LOG_DEBUG;
-		;;
-	    "CRITICAL")
-		log_level=$LOG_CRITICAL;
-		;;
-	    "WARNING")
-		log_level=$LOG_WARNING;
-		;;
-	    "TRACE")
-		log_level=$LOG_TRACE;
-		;;
-	    "NONE")
-		log_level=$LOG_NONE;
-		;;
-	    *)
-		echo "invalid log level $log_level_str, using INFO";
-		log_level=$LOG_INFO;
-		;;
-	esac
-    fi
-    if [ -n "$log_level" ]; then
-        cmd_line=$(echo "$cmd_line --log-level=$log_level");
+            "DEBUG")
+                log_level=$LOG_DEBUG;
+                ;;
+            "CRITICAL")
+                log_level=$LOG_CRITICAL;
+                ;;
+            "WARNING")
+                log_level=$LOG_WARNING;
+                ;;
+            "TRACE")
+                log_level=$LOG_TRACE;
+                ;;
+            "NONE")
+                log_level=$LOG_NONE;
+                ;;
+            *)
+                echo "invalid log level $log_level_str, using INFO";
+                log_level=$LOG_INFO;
+                ;;
+        esac
     fi
 
+    # options without values start here
     if [ -n "$read_only" ]; then
-	cmd_line=$(echo "$cmd_line --read-only");
+        cmd_line=$(echo "$cmd_line --read-only");
     fi
 
     if [ -n "$acl" ]; then
-	cmd_line=$(echo "$cmd_line --acl");
+        cmd_line=$(echo "$cmd_line --acl");
+    fi
+
+    if [ -n "$selinux" ]; then
+         cmd_line=$(echo "$cmd_line --selinux");
+    fi
+
+    if [ -n "$enable_ino32" ]; then
+        cmd_line=$(echo "$cmd_line --enable-ino32");
     fi
 
     if [ -n "$worm" ]; then
         cmd_line=$(echo "$cmd_line --worm");
     fi
+    if [ -n "$volfile_max_fetch_attempts" ]; then
+       cmd_line=$(echo "$cmd_line --volfile-max-fetch-attempts=$volfile_max_fetch_attempts")
+    fi
 
-    if [ -n "$log_file" ]; then
-	cmd_line=$(echo "$cmd_line --log-file=$log_file");
+    if [ -n "$fopen_keep_cache" ]; then
+        cmd_line=$(echo "$cmd_line --fopen-keep-cache");
     fi
 
     if [ -n "$volfile_check" ]; then
-	cmd_line=$(echo "$cmd_line --volfile-check");
+        cmd_line=$(echo "$cmd_line --volfile-check");
+    fi
+
+    if [ -n "$mem_accounting" ]; then
+        cmd_line=$(echo "$cmd_line --mem-accounting");
+    fi
+
+    if [ -n "$aux_gfid_mount" ]; then
+        cmd_line=$(echo "$cmd_line --aux-gfid-mount");
+    fi
+
+    if [ -n "$no_root_squash" ]; then
+        cmd_line=$(echo "$cmd_line --no-root-squash");
+    fi
+
+#options with values start here
+    if [ -n "$log_level" ]; then
+        cmd_line=$(echo "$cmd_line --log-level=$log_level");
+    fi
+
+    if [ -n "$log_file" ]; then
+        cmd_line=$(echo "$cmd_line --log-file=$log_file");
     fi
 
     if [ -n "$direct_io_mode" ]; then
-	cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
+        cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
+    fi
+
+    if [ -n "$use_readdirp" ]; then
+        cmd_line=$(echo "$cmd_line --use-readdirp=$use_readdirp");
     fi
 
     if [ -n "$volume_name" ]; then
         cmd_line=$(echo "$cmd_line --volume-name=$volume_name");
     fi
 
-    if [ -n "$log_server" ]; then
-        if [ -n "$log_server_port" ]; then
-            cmd_line=$(echo "$cmd_line \
---log-server=$log_server \
---log-server-port=$log_server_port");
-        fi
+    if [ -n "$attribute_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --attribute-timeout=$attribute_timeout");
+    fi
+
+    if [ -n "$entry_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --entry-timeout=$entry_timeout");
     fi
 
+    if [ -n "$negative_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --negative-timeout=$negative_timeout");
+    fi
+
+    if [ -n "$gid_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --gid-timeout=$gid_timeout");
+    fi
+
+    if [ -n "$bg_qlen" ]; then
+        cmd_line=$(echo "$cmd_line --background-qlen=$bg_qlen");
+    fi
+
+    if [ -n "$cong_threshold" ]; then
+        cmd_line=$(echo "$cmd_line --congestion-threshold=$cong_threshold");
+    fi
+
+    if [ -n "$fuse_mountopts" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-mountopts=$fuse_mountopts");
+    fi
+
+    if [ -n "$xlator_option" ]; then
+        cmd_line=$(echo "$cmd_line --xlator-option=$xlator_option");
+    fi
+
+    # for rdma volume, we have to fetch volfile with '.rdma' added
+    # to volume name, so that it fetches the right client vol file
+    volume_id_rdma="";
+
     if [ -z "$volfile_loc" ]; then
         if  [ -n "$server_ip" ]; then
+
+            servers=$(parse_volfile_servers ${server_ip});
+            if [ -n "$servers" ]; then
+                for i in $(echo ${servers}); do
+                    cmd_line=$(echo "$cmd_line --volfile-server=$i");
+                done
+            else
+                echo "ERROR: No valid servers found on command line.. exiting"
+                print_usage
+                exit 1
+            fi
+
+            if [ -n "$backupvolfile_server" ]; then
+                if [ -z "$backup_volfile_servers" ]; then
+                    is_valid_hostname ${backupvolfile_server};
+                    if [ $? -eq 1 ]; then
+                        echo "ERROR: Invalid backup server specified.. exiting"
+                        exit 1
+                    fi
+                    cmd_line=$(echo "$cmd_line --volfile-server=$backupvolfile_server");
+                fi
+            fi
+
+            if [ -n "$backup_volfile_servers" ]; then
+                backup_servers=$(parse_backup_volfile_servers ${backup_volfile_servers})
+                for i in $(echo ${backup_servers}); do
+                    cmd_line=$(echo "$cmd_line --volfile-server=$i");
+                done
+            fi
+
             if [ -n "$server_port" ]; then
                 cmd_line=$(echo "$cmd_line --volfile-server-port=$server_port");
             fi
-	    if [ -n "$transport" ]; then
+
+            if [ -n "$transport" ]; then
                 cmd_line=$(echo "$cmd_line --volfile-server-transport=$transport");
+                if [ "$transport" = "rdma" ]; then
+                    volume_id_rdma=".rdma";
+                fi
             fi
+
             if [ -n "$volume_id" ]; then
+                if [ -n "$volume_id_rdma" ]; then
+                    volume_id="$volume_id$volume_id_rdma";
+                fi
                 cmd_line=$(echo "$cmd_line --volfile-id=$volume_id");
             fi
-
-            if [ -n "$backupvolfile_server" ]; then
-                cmd_line1=$(echo "$cmd_line --volfile-server=$backupvolfile_server");
-            fi
-            if [ -n "$volfile_max_fetch_attempts" ]; then
-                cmd_line=$(echo "$cmd_line --volfile-max-fetch-attempts=$volfile_max_fetch_attempts");
-            fi
-	    cmd_line=$(echo "$cmd_line --volfile-server=$server_ip");
         fi
     else
         cmd_line=$(echo "$cmd_line --volfile=$volfile_loc");
     fi
 
-    cmd_line=$(echo "$cmd_line $mount_point");
-    err=0;
-    $cmd_line;
+    if [ -n "$fuse_mountopts" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-mountopts=$fuse_mountopts");
+    fi
 
+    cmd_line=$(echo "$cmd_line $mount_point");
 
-    inode=$(stat -c %i $mount_point 2>/dev/null);
+    $cmd_line;
 
+    inode=$( ${getinode} $mount_point 2>/dev/null);
     # this is required if the stat returns error
-    if [ -z "$inode" ]; then
-        inode="0";
-    fi
-
-    # retry the failover
-    # if [ $? != "0" ]; then # <--- TODO: Once glusterfs returns proper error code, change it.
-    if [ $inode -ne 1 ]; then
-        err=1;
-        if [ -n "$cmd_line1" ]; then
-            cmd_line1=$(echo "$cmd_line1 $mount_point");
-            $cmd_line1;
-            err=0;
-
-            inode=$(stat -c %i $mount_point 2>/dev/null);
-            # this is required if the stat returns error
-            if [ -z "$inode" ]; then
-                inode="0";
-            fi
-            if [ $inode -ne 1 ]; then
-                err=1;
-            fi
-        fi
-    fi
-
-    if [ $err -eq "1" ]; then
-            echo "Mount failed. Please check the log file for more details."
-            umount $mount_point > /dev/null 2>&1;
-            exit 1;
+    if [ $? -ne 0 ]; then
+        echo "Mount failed. Please check the log file for more details."
+        umount $mount_point > /dev/null 2>&1;
+        exit 1;
     fi
 }
 
-usage ()
+print_usage ()
 {
-
-echo "Usage:  mount.glusterfs <volumeserver>:<volumeid/volumeport> -o <options> <mountpoint>
+cat << EOF
+Usage: $0 <volumeserver>:<volumeid/volumeport> -o<options> <mountpoint>
 Options:
-man 8 mount.glusterfs
-
-To display the version number of the mount helper:
-mount.glusterfs --version"
-
+man 8 $0
+To display the version number of the mount helper: $0 -V
+EOF
 }
 
 # check for recursive mounts. i.e, mounting over an existing brick
 check_recursive_mount ()
 {
-    if [ $2 = "/" ]; then
-        echo Cannot mount over root;
+    if [ $1 = "/" ]; then
+        echo "Cannot mount over root";
         exit 2;
     fi
+
     # GFID check first
     # remove trailing / from mount point
-    mnt_dir=${2%/};
-
-    export PATH;
-    # check whether getfattr exists
-    which getfattr > /dev/null 2>&1;
-    if [ $? -ne 0 ]; then
-        return;
-    fi
+    mnt_dir=${1%/};
 
-    getfattr -n trusted.gfid $mnt_dir 2>/dev/null | grep -iq "trusted.gfid=";
-    if [ $? -eq 0 ]; then
-        echo "ERROR: $mnt_dir is in use as a brick of a gluster volume";
-        exit 2;
+    if [ -n ${getfattr} ]; then
+        ${getfattr} -n trusted.gfid $mnt_dir 2>/dev/null | grep -iq "trusted.gfid=";
+        if [ $? -eq 0 ]; then
+            echo "ERROR: $mnt_dir is in use as a brick of a gluster volume";
+            exit 2;
+        fi
     fi
 
     # check if the mount point is a brick's parent directory
-    GLUSTERD_WORKDIR="/etc/glusterd";
+    GLUSTERD_WORKDIR="/var/lib/glusterd";
 
-    ls -L "$GLUSTERD_WORKDIR"/vols/*/bricks/* > /dev/null 2>&1;
+    lsL "$GLUSTERD_WORKDIR"/vols/*/bricks/* > /dev/null 2>&1;
     if [ $? -ne 0 ]; then
         return;
     fi
 
-    brick_path=(`grep ^path "$GLUSTERD_WORKDIR"/vols/*/bricks/* | cut -d "=" -f 2`);
-    root_inode=`stat -Lc %i /`;
-    root_dev=`stat -Lc %d /`;
-    mnt_inode=`stat -Lc %i $mnt_dir`;
-    mnt_dev=`stat -Lc %d $mnt_dir`;
-    for brick in "$brick_path";
-    do
+    brick_path=`grep ^path "$GLUSTERD_WORKDIR"/vols/*/bricks/* 2>/dev/null | cut -d "=" -f 2`;
+    root_inode=`${lgetinode} /`;
+    root_dev=`${lgetdev} /`;
+    mnt_inode=`${lgetinode} $mnt_dir`;
+    mnt_dev=`${lgetdev} $mnt_dir`;
+    for brick in "$brick_path"; do
         # evaluate brick path to see if this is local, if non-local, skip iteration
         ls $brick > /dev/null 2>&1;
         if [ $? -ne 0 ]; then
             continue;
         fi
-        getfattr -n trusted.gfid "$brick" 2>/dev/null | grep -iq "trusted.gfid=";
-        if [ $? -ne 0 ]; then
-            continue;
+
+        if [ -n ${getfattr} ]; then
+            ${getfattr} -n trusted.gfid "$brick" 2>/dev/null | grep -iq "trusted.gfid=";
+            if [ $? -eq 0 ]; then
+                # brick is local
+                while [ 1 ]; do
+                    tmp_brick="$brick";
+                    brick="$brick"/..;
+                    brick_dev=`${lgetdev} $brick`;
+                    brick_inode=`${lgetinode} $brick`;
+                    if [ "$mnt_inode" -eq "$brick_inode" \
+                        -a "$mnt_dev" -eq "$brick_dev" ]; then
+                        echo "ERROR: ${mnt_dir} is a parent of the brick ${tmp_brick}";
+                        exit 2;
+                    fi
+                    [ "$root_inode" -ne "$brick_inode" \
+                        -o "$root_dev" -ne "$brick_dev" ] || break;
+                done;
+            else
+                continue;
+            fi
         else
-            # brick is local
-            while [ 1 ];
-            do
-                tmp_brick="$brick";
-                brick="$brick"/..;
-                brick_dev=`stat -Lc %d $brick`;
-                brick_inode=`stat -Lc %i $brick`;
-                if [ "$mnt_inode" -eq "$brick_inode" -a "$mnt_dev" -eq "$brick_dev" ]; then
-                    echo ERROR: $mnt_dir is a parent of the brick $tmp_brick;
-                    exit 2;
-                fi
-                [ "$root_inode" -ne "$brick_inode" -o "$root_dev" -ne "$brick_dev" ] || break;
-            done;
+            continue;
         fi
     done;
 }
 
-main ()
+with_options()
 {
-    helper=$(echo "$@" | sed -n 's/.*\--[ ]*\([^ ]*\).*/\1/p');
+    local key=$1
+    local value=$2
+
+    # Handle options with values.
+    case "$key" in
+        "log-level")
+            log_level_str=$value
+            ;;
+        "log-file")
+            log_file=$value
+            ;;
+        "transport")
+            transport=$value
+            ;;
+        "direct-io-mode")
+            direct_io_mode=$value
+            ;;
+        "volume-name")
+            volume_name=$value
+            ;;
+        "volume-id")
+            volume_id=$value
+            ;;
+        "volfile-check")
+            volfile_check=$value
+            ;;
+        "server-port")
+            server_port=$value
+            ;;
+        "attribute-timeout")
+            attribute_timeout=$value
+            ;;
+        "entry-timeout")
+            entry_timeout=$value
+            ;;
+        "negative-timeout")
+            negative_timeout=$value
+            ;;
+        "gid-timeout")
+            gid_timeout=$value
+            ;;
+        "background-qlen")
+            bg_qlen=$value
+            ;;
+        "backup-volfile-servers")
+            backup_volfile_servers=$value
+            ;;
+        "backupvolfile-server")
+            backupvolfile_server=$value
+            ;;
+        "fetch-attempts")
+            volfile_max_fetch_attempts=$value
+            ;;
+        "congestion-threshold")
+            cong_threshold=$value
+            ;;
+        "xlator-option")
+            xlator_option=$value
+            ;;
+        "fuse-mountopts")
+            fuse_mountopts=$value
+            ;;
+        "use-readdirp")
+            use_readdirp=$value
+            ;;
+        "no-root-squash")
+            if [ $value == "yes" ] ||
+                [ $value == "on" ] ||
+                [ $value == "enable" ] ||
+                [ $value == "true" ] ; then
+                no_root_squash=1;
+            fi ;;
+        "root-squash")
+            if [ $value == "no" ] ||
+                [ $value == "off" ] ||
+                [ $value == "disable" ] ||
+                [ $value == "false" ] ; then
+                no_root_squash=1;
+            fi ;;
+        *)
+            echo "Invalid option: $key"
+            exit 0
+            ;;
+    esac
+}
 
-    options=$(echo "$@" | sed -n 's/.*\-o[ ]*\([^ ]*\).*/\1/p');
+without_options()
+{
+    local option=$1
+    # Handle options without values.
+    case "$option" in
+        "ro")
+            read_only=1
+            ;;
+        "acl")
+            acl=1
+            ;;
+        "selinux")
+            selinux=1
+            ;;
+        "worm")
+            worm=1
+            ;;
+        "fopen-keep-cache")
+            fopen_keep_cache=1
+            ;;
+        "enable-ino32")
+            enable_ino32=1
+            ;;
+        "mem-accounting")
+            mem_accounting=1
+            ;;
+        "aux-gfid-mount")
+            if [ ${uname_s} = "Linux" ]; then
+                aux_gfid_mount=1
+            fi
+            ;;
+         # "mount -t glusterfs" sends this, but it's useless.
+        "rw")
+            ;;
+         # these ones are interpreted during system initialization
+        "noauto")
+            ;;
+        "_netdev")
+            ;;
+        *)
+            echo "Invalid option $option";
+            exit 0
+            ;;
+    esac
+}
 
-    new_log_level=$(echo "$options" | sed -n 's/.*log-level=\([^,]*\).*/\1/p');
+parse_options()
+{
+    local optarg=${1}
+    for pair in $(echo ${optarg//,/ }); do
+        key=$(echo "$pair" | cut -f1 -d'=');
+        value=$(echo "$pair" | cut -f2- -d'=');
+        if [ "$key" = "$value" ]; then
+            without_options $pair;
+        else
+            with_options $key $value;
+        fi
+    done
+}
 
-    [ -n "$new_log_level" ] && {
-	log_level_str="$new_log_level";
+update_updatedb()
+{
+    # Append fuse.glusterfs to PRUNEFS variable in updatedb.conf(5).
+    # updatedb(8) should not index files under GlusterFS, indexing
+    # GlusterFS is not necessary and should be avoided.
+    # Following code disables updatedb crawl on 'glusterfs'
+    test -f $UPDATEDBCONF && {
+        if ! grep -q 'glusterfs' $UPDATEDBCONF; then
+            sed 's/\(PRUNEFS.*\)"/\1 fuse.glusterfs"/' $UPDATEDBCONF \
+                > ${UPDATEDBCONF}.bak
+            mv -f ${UPDATEDBCONF}.bak $UPDATEDBCONF
+        fi
     }
+}
 
-    log_file=$(echo "$options" | sed -n 's/.*log-file=\([^,]*\).*/\1/p');
-
-    read_only=$(echo "$options" | sed -n 's/.*\(ro\)[^,]*.*/\1/p');
-
-    acl=$(echo "$options" | sed -n 's/.*\(acl\)[^,]*.*/\1/p');
-
-    worm=$(echo "$options" | sed -n 's/.*\(worm\)[^,]*.*/\1/p');
-
-    transport=$(echo "$options" | sed -n 's/.*transport=\([^,]*\).*/\1/p');
-
-    direct_io_mode=$(echo "$options" | sed -n 's/.*direct-io-mode=\([^,]*\).*/\1/p');
-
-    volume_name=$(echo "$options" | sed -n 's/.*volume-name=\([^,]*\).*/\1/p');
-
-    volume_id=$(echo "$options" | sed -n 's/.*volume_id=\([^,]*\).*/\1/p');
-
-    volfile_check=$(echo "$options" | sed -n 's/.*volfile-check=\([^,]*\).*/\1/p');
-
-    volfile_max_fetch_attempts=$(echo "$options" | sed -n 's/.*fetch-attempts=\([^,]*\).*/\1/p');
-
-    server_port=$(echo "$options" | sed -n 's/.*server-port=\([^,]*\).*/\1/p');
-
-    backupvolfile_server=$(echo "$options" | sed -n 's/.*backupvolfile-server=\([^,]*\).*/\1/p');
-
-    log_server=$(echo "$options" | sed -n 's/.*log-server=\([^,]*\).*/\1/p');
+main ()
+{
 
-    log_server_port=$(echo "$options" | sed -n 's/.*log-server-port=\([^,]*\).*/\1/p');
+    volfile_loc=$1
+    mount_point=$2
 
-    volfile_loc="$1";
+    ## `mount` specifies options as a last argument
+    shift 2;
+    while getopts "Vo:h" opt; do
+        case "${opt}" in
+            o)
+                parse_options ${OPTARG};
+                ;;
+            V)
+                ${cmd_line} -V;
+                exit 0;
+                ;;
+            h)
+                print_usage;
+                exit 0;
+                ;;
+            ?)
+                print_usage;
+                exit 0;
+                ;;
+        esac
+    done
 
     [ -r "$volfile_loc" ] || {
         server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
-        test_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
-        [ -n "$test_str" ] && {
-            volume_id="$test_str";
+        volume_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
+        [ -n "$volume_str" ] && {
+            volume_id="$volume_str";
         }
-	volfile_loc="";
+        volfile_loc="";
     }
 
-    new_fs_options=$(echo "$options" | sed -e 's/[,]*log-file=[^,]*//' \
-        -e 's/[,]*log-level=[^,]*//' \
-        -e 's/[,]*volume-name=[^,]*//' \
-        -e 's/[,]*direct-io-mode=[^,]*//' \
-        -e 's/[,]*volfile-check=[^,]*//' \
-        -e 's/[,]*transport=[^,]*//' \
-        -e 's/[,]*backupvolfile-server=[^,]*//' \
-        -e 's/[,]*server-port=[^,]*//' \
-        -e 's/[,]*volume-id=[^,]*//' \
-        -e 's/[,]*fetch-attempts=[^,]*//' \
-        -e 's/[,]*log-server=[^,]*//' \
-        -e 's/[,]*ro[^,]*//' \
-        -e 's/[,]*acl[^,]*//' \
-        -e 's/[,]*worm[^,]*//' \
-        -e 's/[,]*log-server-port=[^,]*//');
-
-    #
-    [ -n "$helper" ] && {
-        cmd_line=$(echo "$cmd_line --$helper");
-        exec $cmd_line;
+    [ -z "$volume_id" -o -z "$server_ip" ] && {
+        cat <<EOF
+ERROR: Server name/volume name unspecified cannot proceed further..
+Please specify correct format
+Usage:
+man 8 $0
+EOF
         exit 0;
     }
 
-    mount_point=""
-    for arg in "$@"; do
-        [ -d "$arg" ] && {
-            mount_point=$arg
-        }
-    done
+    grep_ret=$(echo ${mount_point} | grep '^\-o');
+    [ "x" != "x${grep_ret}" ] && {
+        cat <<EOF
+ERROR: -o options cannot be specified in either first two arguments..
+Please specify correct style
+Usage:
+man 8 $0
+EOF
+        exit 0;
+    }
 
     # No need to do a ! -d test, it is taken care while initializing the
     # variable mount_point
-    [ -z "$mount_point" ] && {
-        echo "ERROR: Mount point does not exist."
-        usage;
+    [ -z "$mount_point" -o ! -d "$mount_point" ] && {
+        cat <<EOF
+ERROR: Mount point does not exist
+Please specify a mount point
+Usage:
+man 8 $0
+EOF
         exit 0;
     }
 
     # Simple check to avoid multiple identical mounts
-    if grep -q " $mount_point fuse" /etc/mtab; then
+    if grep -q "[[:space:]+]${mount_point}[[:space:]+]fuse" $mounttab; then
         echo -n "$0: according to mtab, GlusterFS is already mounted on "
         echo "$mount_point"
         exit 0;
     fi
 
-    check_recursive_mount "$@";
-
-    fs_options=$(echo "$fs_options,$new_fs_options");
+    check_recursive_mount "$mount_point";
 
-    # Append fuse.glusterfs to PRUNEFS variable in updatedb.conf(5). updatedb(8)
-    # should not index files under GlusterFS, indexing will slow down GlusteFS
-    # if the filesystem is several TB in size.
-    test -f $UPDATEDBCONF && {
-        if ! grep -q 'glusterfs' $UPDATEDBCONF; then
-            sed 's/\(PRUNEFS.*\)"/\1 fuse.glusterfs"/' $UPDATEDBCONF \
-                > ${UPDATEDBCONF}.bak
-            mv -f ${UPDATEDBCONF}.bak $UPDATEDBCONF
-        fi
-    }
+    update_updatedb;
 
     start_glusterfs;
 }
 
 _init "$@" && main "$@";
-
diff --git a/xlators/mount/fuse/utils/mount_glusterfs.in b/xlators/mount/fuse/utils/mount_glusterfs.in
index 3d4d28620..b12b4e04e 100755
--- a/xlators/mount/fuse/utils/mount_glusterfs.in
+++ b/xlators/mount/fuse/utils/mount_glusterfs.in
@@ -85,7 +85,7 @@ start_glusterfs ()
 --volfile-server-transport=$transport");
 	else
 	    cmd_line=$(echo "$cmd_line \
---volfile-server=$server_ip \
+--volfile-server=$server_ip");
 	fi
     else
 	cmd_line=$(echo "$cmd_line --volfile=$volfile_loc");
@@ -173,6 +173,7 @@ main ()
     
     [ -r "$volfile_loc" ] || {
         server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        volume_id=$(echo "$volfile_loc" | sed -n 's/[a-zA-Z0-9:.\-]*:\(.*\)/\1/p');
 	volfile_loc="";
     }
     # following line is product of love towards sed
diff --git a/xlators/nfs/server/src/Makefile.am b/xlators/nfs/server/src/Makefile.am
index a2c76a502..62fbf6535 100644
--- a/xlators/nfs/server/src/Makefile.am
+++ b/xlators/nfs/server/src/Makefile.am
@@ -1,17 +1,24 @@
 xlator_LTLIBRARIES = server.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/nfs
 nfsrpclibdir = $(top_srcdir)/rpc/rpc-lib/src
-server_la_LDFLAGS = -module -avoidversion
-server_la_SOURCES = nfs.c nfs-common.c nfs-fops.c nfs-inodes.c nfs-generics.c mount3.c nfs3-fh.c nfs3.c nfs3-helpers.c nlm4.c nlmcbk_svc.c
+server_la_LDFLAGS = -module -avoid-version
+server_la_SOURCES = nfs.c nfs-common.c nfs-fops.c nfs-inodes.c \
+	nfs-generics.c mount3.c nfs3-fh.c nfs3.c nfs3-helpers.c nlm4.c \
+	nlmcbk_svc.c mount3udp_svc.c acl3.c
 server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = nfs.h nfs-common.h nfs-fops.h nfs-inodes.h nfs-generics.h mount3.h nfs3-fh.h nfs3.h nfs3-helpers.h nfs-mem-types.h nlm4.h
+noinst_HEADERS = nfs.h nfs-common.h nfs-fops.h nfs-inodes.h nfs-generics.h \
+	mount3.h nfs3-fh.h nfs3.h nfs3-helpers.h nfs-mem-types.h nlm4.h \
+	acl3.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)\
-	-I$(nfsrpclibdir) -L$(xlatordir)/ -I$(CONTRIBDIR)/rbtree\
-	-I$(top_srcdir)/rpc/xdr/src/ \
-	-I$(top_srcdir)/contrib/md5/
+	-I$(top_srcdir)/libglusterfs/src \
+	-I$(nfsrpclibdir) -I$(CONTRIBDIR)/rbtree \
+	-I$(top_srcdir)/rpc/xdr/src/
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+AM_LDFLAGS = -L$(xlatordir)
 
 CLEANFILES =
diff --git a/xlators/nfs/server/src/acl3.c b/xlators/nfs/server/src/acl3.c
new file mode 100644
index 000000000..43156eb44
--- /dev/null
+++ b/xlators/nfs/server/src/acl3.c
@@ -0,0 +1,843 @@
+/*
+ * Copyright (c) 2012-2013 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "rpcsvc.h"
+#include "dict.h"
+#include "xlator.h"
+#include "nfs.h"
+#include "mem-pool.h"
+#include "logging.h"
+#include "nfs-fops.h"
+#include "inode.h"
+#include "nfs3.h"
+#include "nfs-mem-types.h"
+#include "nfs3-helpers.h"
+#include "nfs3-fh.h"
+#include "nfs-generics.h"
+#include "acl3.h"
+#include "byte-order.h"
+
+static int
+acl3_nfs_acl_to_xattr (aclentry *ace, void *xattrbuf,
+                       int aclcount, int defacl);
+
+static int
+acl3_nfs_acl_from_xattr (aclentry *ace, void *xattrbuf,
+                         int bufsize, int defacl);
+
+typedef ssize_t (*acl3_serializer) (struct iovec outmsg, void *args);
+
+extern void nfs3_call_state_wipe (nfs3_call_state_t *cs);
+
+extern nfs3_call_state_t *
+nfs3_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
+
+extern int
+nfs3_fh_validate (struct nfs3_fh *fh);
+
+extern fattr3
+nfs3_stat_to_fattr3 (struct iatt *buf);
+
+#define acl3_validate_nfs3_state(request, state, status, label, retval) \
+        do      {                                                       \
+                state = rpcsvc_request_program_private (request);       \
+                if (!state) {                                           \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "NFSv3 state "    \
+                                "missing from RPC request");            \
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
+                        status = NFS3ERR_SERVERFAULT;                   \
+                        goto label;                                     \
+                }                                                       \
+        } while (0);                                                    \
+
+#define acl3_validate_gluster_fh(handle, status, errlabel)              \
+        do {                                                            \
+                if (!nfs3_fh_validate (handle)) {                       \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Bad Handle");    \
+                        status = NFS3ERR_BADHANDLE;                     \
+                        goto errlabel;                                  \
+                }                                                       \
+        } while (0)                                                     \
+
+
+extern xlator_t *
+nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh);
+
+#define acl3_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
+        do {                                                            \
+                char exportid[256], gfid[256];                          \
+                rpc_transport_t *trans = NULL;                          \
+                volume = nfs3_fh_to_xlator ((nfs3state), handle);      \
+                if (!volume) {                                          \
+                        uuid_unparse (handle->exportid, exportid);       \
+                        uuid_unparse (handle->gfid, gfid);               \
+                        trans = rpcsvc_request_transport (req);         \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Failed to map "  \
+                                "FH to vol: client=%s, exportid=%s, gfid=%s",\
+                                trans->peerinfo.identifier, exportid,   \
+                                gfid);                                  \
+                        gf_log (GF_ACL, GF_LOG_ERROR,                   \
+                                "Stale nfs client %s must be trying to "\
+                                "connect to a deleted volume, please "  \
+                                "unmount it.", trans->peerinfo.identifier);\
+                        status = NFS3ERR_STALE;                         \
+                        goto label;                                     \
+                } else {                                                \
+                        gf_log (GF_ACL, GF_LOG_TRACE, "FH to Volume: %s"\
+                                ,volume->name);                         \
+                        rpcsvc_request_set_private (req, volume);       \
+                }                                                       \
+        } while (0);                                                    \
+
+#define acl3_volume_started_check(nfs3state, vlm, rtval, erlbl)         \
+        do {                                                            \
+              if ((!nfs_subvolume_started (nfs_state (nfs3state->nfsx), vlm))){\
+                      gf_log (GF_ACL, GF_LOG_ERROR, "Volume is disabled: %s",\
+                              vlm->name);                               \
+                      rtval = RPCSVC_ACTOR_IGNORE;                      \
+                      goto erlbl;                                       \
+              }                                                         \
+        } while (0)                                                     \
+
+#define acl3_check_fh_resolve_status(cst, nfstat, erlabl)               \
+        do {                                                            \
+                xlator_t *xlatorp = NULL;                               \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
+                if ((cst)->resolve_ret < 0) {                           \
+                        trans = rpcsvc_request_transport (cst->req);    \
+                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
+                                                     &cst->resolvefh);  \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,           \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid);                                \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Unable to resolve FH"\
+                                ": %s", buf);                           \
+                        nfstat = nfs3_errno_to_nfsstat3 (cst->resolve_errno);\
+                        goto erlabl;                                    \
+                }                                                       \
+        } while (0)                                                     \
+
+#define acl3_handle_call_state_init(nfs3state, calls, rq, v, opstat, errlabel)\
+        do {                                                            \
+                calls = nfs3_call_state_init ((nfs3state), (rq), v); \
+                if (!calls) {                                           \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Failed to "      \
+                                "init call state");                     \
+                        opstat = NFS3ERR_SERVERFAULT;                   \
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
+                        goto errlabel;                                  \
+                }                                                       \
+        } while (0)                                                     \
+
+
+int
+acl3svc_submit_reply (rpcsvc_request_t *req, void *arg, acl3_serializer sfunc)
+{
+        struct iovec            outmsg  = {0, };
+        struct iobuf            *iob    = NULL;
+        struct nfs3_state       *nfs3   = NULL;
+        int                     ret     = -1;
+        ssize_t                 msglen  = 0;
+        struct iobref           *iobref = NULL;
+
+        if (!req)
+                return -1;
+
+        nfs3 = (struct nfs3_state *)rpcsvc_request_program_private (req);
+        if (!nfs3) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "mount state not found");
+                goto ret;
+        }
+
+        /* First, get the io buffer into which the reply in arg will
+         * be serialized.
+         */
+        iob = iobuf_get (nfs3->iobpool);
+        if (!iob) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to get iobuf");
+                goto ret;
+        }
+
+        iobuf_to_iovec (iob, &outmsg);
+        /* Use the given serializer to translate the give C structure in arg
+         * to XDR format which will be written into the buffer in outmsg.
+         */
+        msglen = sfunc (outmsg, arg);
+        if (msglen < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+        outmsg.iov_len = msglen;
+
+        iobref = iobref_new ();
+        if (iobref == NULL) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to get iobref");
+                goto ret;
+        }
+
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
+
+        /* Then, submit the message for transmission. */
+        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Reply submission failed");
+                goto ret;
+        }
+
+        ret = 0;
+ret:
+        if (iob)
+                iobuf_unref (iob);
+        if (iobref)
+                iobref_unref (iobref);
+
+        return ret;
+}
+
+
+int
+acl3svc_null (rpcsvc_request_t *req)
+{
+        struct iovec    dummyvec = {0, };
+
+        if (!req) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Got NULL request!");
+                return 0;
+        }
+        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
+        return 0;
+}
+
+int
+acl3_getacl_reply (rpcsvc_request_t *req, getaclreply *reply)
+{
+        acl3svc_submit_reply (req, (void *)reply,
+                              (acl3_serializer)xdr_serialize_getaclreply);
+        return 0;
+}
+
+int
+acl3_setacl_reply (rpcsvc_request_t *req, setaclreply *reply)
+{
+        acl3svc_submit_reply (req, (void *)reply,
+                              (acl3_serializer)xdr_serialize_setaclreply);
+        return 0;
+}
+
+
+int
+acl3_getacl_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict,
+                 dict_t *xdata)
+{
+        nfsstat3                 stat = NFS3ERR_SERVERFAULT;
+        nfs3_call_state_t        *cs = NULL;
+        data_t                   *data = NULL;
+        getaclreply              *getaclreply = NULL;
+        int                      aclcount = 0;
+        int                      defacl = 1; /* DEFAULT ACL */
+
+        if (!frame->local) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
+                       " frame->local NULL");
+                return  EINVAL;
+        }
+        cs = frame->local;
+        getaclreply = &cs->args.getaclreply;
+        if (op_ret < 0) {
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
+                goto err;
+        }
+
+        getaclreply->aclentry.aclentry_val = cs->aclentry;
+        getaclreply->daclentry.daclentry_val = cs->daclentry;
+
+        /* getfacl: NFS USER ACL */
+        data = dict_get (dict, POSIX_ACL_ACCESS_XATTR);
+        if (data && data->data) {
+                aclcount = acl3_nfs_acl_from_xattr (cs->aclentry,
+                                                    data->data,
+                                                    data->len,
+                                                    !defacl);
+                if (aclcount < 0) {
+                        gf_log (GF_ACL, GF_LOG_ERROR,
+                                "Failed to get USER ACL");
+                        stat = nfs3_errno_to_nfsstat3 (-aclcount);
+                        goto err;
+                }
+
+                getaclreply->aclcount = aclcount;
+                getaclreply->aclentry.aclentry_len = aclcount;
+        }
+
+        /* getfacl: NFS DEFAULT ACL */
+        data = dict_get (dict, POSIX_ACL_DEFAULT_XATTR);
+        if (data && data->data) {
+                aclcount = acl3_nfs_acl_from_xattr (cs->daclentry,
+                                                    data->data,
+                                                    data->len,
+                                                    defacl);
+                if (aclcount < 0) {
+                        gf_log (GF_ACL, GF_LOG_ERROR,
+                                "Failed to get DEFAULT ACL");
+                        stat = nfs3_errno_to_nfsstat3 (-aclcount);
+                        goto err;
+                }
+
+                getaclreply->daclcount = aclcount;
+                getaclreply->daclentry.daclentry_len = aclcount;
+        }
+
+        acl3_getacl_reply (cs->req, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+
+err:
+        if (getaclreply)
+                getaclreply->status = stat;
+        acl3_getacl_reply (cs->req, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+acl3_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs3_call_state_t              *cs = NULL;
+        getaclreply                     *getaclreply = NULL;
+        int                             ret = -1;
+        nfs_user_t                      nfu = {0, };
+        uint64_t                        deviceid = 0;
+
+        if (!frame->local) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
+                       " frame->local NULL");
+                return EINVAL;
+        }
+
+        cs = frame->local;
+        getaclreply = &cs->args.getaclreply;
+
+        if (op_ret == -1) {
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
+                goto err;
+        }
+
+        /* Fill the attrs before xattrs */
+        getaclreply->attr_follows = TRUE;
+        deviceid = nfs3_request_xlator_deviceid (cs->req);
+        nfs3_map_deviceid_to_statdev (buf, deviceid);
+        getaclreply->attr = nfs3_stat_to_fattr3 (buf);
+
+        nfs_request_user_init (&nfu, cs->req);
+        ret = nfs_getxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                            NULL, NULL, acl3_getacl_cbk, cs);
+        if (ret < 0) {
+                stat = nfs3_errno_to_nfsstat3 (-ret);
+                goto err;
+        }
+        return 0;
+err:
+        getaclreply->status = stat;
+        acl3_getacl_reply (cs->req, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+
+int
+acl3_getacl_resume (void *carg)
+{
+        int                             ret = -1;
+        nfs3_call_state_t               *cs = NULL;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs_user_t                      nfu = {0, };
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        acl3_check_fh_resolve_status (cs, stat, acl3err);
+        nfs_request_user_init (&nfu, cs->req);
+
+        ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                        acl3_stat_cbk, cs);
+        stat = -ret;
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to open_and_resume");
+                cs->args.getaclreply.status = nfs3_errno_to_nfsstat3 (stat);
+                acl3_getacl_reply (cs->req, &cs->args.getaclreply);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+
+int
+acl3svc_getacl (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        struct nfs3_fh                  fh, *fhp = NULL;
+        getaclargs                      getaclargs;
+        getaclreply                     getaclreply;
+
+        if (!req)
+                return ret;
+
+        acl3_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        memset (&getaclargs, 0, sizeof (getaclargs));
+        memset (&getaclreply, 0, sizeof (getaclreply));
+        getaclargs.fh.n_bytes = (char *)&fh;
+        if (xdr_to_getaclargs(req->msg[0], &getaclargs) <= 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        /* Validate ACL mask */
+        if (getaclargs.mask & ~(NFS_ACL|NFS_ACLCNT|NFS_DFACL|NFS_DFACLCNT)) {
+                stat = NFS3ERR_INVAL;
+                goto acl3err;
+        }
+
+        fhp = &fh;
+        acl3_validate_gluster_fh (&fh, stat, acl3err);
+        acl3_map_fh_to_volume (nfs->nfs3state, fhp, req, vol, stat, acl3err);
+        acl3_volume_started_check (nfs3, vol, ret, rpcerr);
+        acl3_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     vol, stat, acl3err);
+
+        cs->vol = vol;
+        cs->args.getaclreply.mask = getaclargs.mask;
+
+        ret = nfs3_fh_resolve_and_resume (cs, fhp, NULL, acl3_getacl_resume);
+        stat = nfs3_errno_to_nfsstat3 (-ret);
+
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to resolve and resume");
+                getaclreply.status = stat;
+                acl3_getacl_reply (req, &getaclreply);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+rpcerr:
+        return ret;
+}
+
+int
+acl3_setacl_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 dict_t *xdata)
+{
+        nfs3_call_state_t               *cs = NULL;
+        cs = frame->local;
+        if (op_ret < 0) {
+                nfsstat3 status = nfs3_cbk_errno_status (op_ret, op_errno);
+                cs->args.setaclreply.status = status;
+        }
+
+        acl3_setacl_reply (cs->req, &cs->args.setaclreply);
+
+        return 0;
+}
+
+int
+acl3_setacl_resume (void *carg)
+{
+        int                             ret = -1;
+        nfs3_call_state_t               *cs = NULL;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs_user_t                      nfu = {0, };
+        dict_t                          *xattr = NULL;
+
+        if (!carg)
+                return ret;
+        cs = (nfs3_call_state_t *)carg;
+        acl3_check_fh_resolve_status (cs, stat, acl3err);
+        nfs_request_user_init (&nfu, cs->req);
+        xattr = dict_new();
+        if (cs->aclcount)
+        ret = dict_set_static_bin (xattr, POSIX_ACL_ACCESS_XATTR,
+                                   cs->aclxattr,
+                                   posix_acl_xattr_size (cs->aclcount));
+        if (cs->daclcount)
+        ret = dict_set_static_bin (xattr, POSIX_ACL_DEFAULT_XATTR,
+                                   cs->daclxattr,
+                                   posix_acl_xattr_size (cs->daclcount));
+
+        ret = nfs_setxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, xattr,
+                            0, NULL, acl3_setacl_cbk, cs);
+        dict_unref (xattr);
+
+acl3err:
+        if (ret < 0) {
+                stat = -ret;
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to open_and_resume");
+                cs->args.setaclreply.status = nfs3_errno_to_nfsstat3 (stat);
+                acl3_setacl_reply (cs->req, &cs->args.setaclreply);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+
+int
+acl3svc_setacl (rpcsvc_request_t *req)
+{
+        xlator_t                       *vol = NULL;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t              *cs = NULL;
+        int                            ret = RPCSVC_ACTOR_ERROR;
+        nfsstat3                       stat = NFS3ERR_SERVERFAULT;
+        struct nfs3_fh                 fh;
+        struct nfs3_fh                 *fhp = NULL;
+        setaclargs                     setaclargs;
+        setaclreply                    setaclreply;
+        aclentry                       *daclentry = NULL;
+        aclentry                       *aclentry = NULL;
+        int                            aclerrno = 0;
+        int                            defacl = 1;
+
+        if (!req)
+                return ret;
+        aclentry =  GF_CALLOC (NFS_ACL_MAX_ENTRIES, sizeof(*aclentry),
+                               gf_nfs_mt_arr);
+        if (!aclentry) {
+                goto rpcerr;
+        }
+        daclentry = GF_CALLOC (NFS_ACL_MAX_ENTRIES, sizeof(*daclentry),
+                               gf_nfs_mt_arr);
+        if (!daclentry) {
+                goto rpcerr;
+        }
+
+        acl3_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        memset (&setaclargs, 0, sizeof (setaclargs));
+        memset (&setaclreply, 0, sizeof (setaclreply));
+        memset (&fh, 0, sizeof (fh));
+        setaclargs.fh.n_bytes = (char *)&fh;
+        setaclargs.aclentry.aclentry_val = aclentry;
+        setaclargs.daclentry.daclentry_val = daclentry;
+        if (xdr_to_setaclargs(req->msg[0], &setaclargs) <= 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        /* Validate ACL mask */
+        if (setaclargs.mask & ~(NFS_ACL|NFS_ACLCNT|NFS_DFACL|NFS_DFACLCNT)) {
+                stat = NFS3ERR_INVAL;
+                goto acl3err;
+        }
+
+        fhp = &fh;
+        acl3_validate_gluster_fh (fhp, stat, acl3err);
+        acl3_map_fh_to_volume (nfs->nfs3state, fhp, req, vol, stat, acl3err);
+        acl3_volume_started_check (nfs3, vol, ret, rpcerr);
+        acl3_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     vol, stat, acl3err);
+
+        cs->vol = vol;
+        cs->aclcount = setaclargs.aclcount;
+        cs->daclcount = setaclargs.daclcount;
+
+        /* setfacl: NFS USER ACL */
+        aclerrno = acl3_nfs_acl_to_xattr (aclentry,
+                                          cs->aclxattr,
+                                          cs->aclcount,
+                                          !defacl);
+        if (aclerrno < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to set USER ACL");
+                stat = nfs3_errno_to_nfsstat3 (-aclerrno);
+                goto acl3err;
+        }
+
+        /* setfacl: NFS DEFAULT ACL */
+        aclerrno = acl3_nfs_acl_to_xattr (daclentry,
+                                          cs->daclxattr,
+                                          cs->daclcount,
+                                          defacl);
+        if (aclerrno < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to set DEFAULT ACL");
+                stat = nfs3_errno_to_nfsstat3 (-aclerrno);
+                goto acl3err;
+        }
+
+        ret = nfs3_fh_resolve_and_resume (cs, fhp, NULL, acl3_setacl_resume);
+        stat = nfs3_errno_to_nfsstat3 (-ret);
+
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to resolve and resume");
+                setaclreply.status = stat;
+                acl3_setacl_reply (req, &setaclreply);
+                nfs3_call_state_wipe (cs);
+                GF_FREE(aclentry);
+                GF_FREE(daclentry);
+                return 0;
+        }
+
+rpcerr:
+        if (ret < 0)
+                nfs3_call_state_wipe (cs);
+        if (aclentry)
+                GF_FREE (aclentry);
+        if (daclentry)
+                GF_FREE (daclentry);
+        return ret;
+}
+
+
+
+rpcsvc_actor_t  acl3svc_actors[ACL3_PROC_COUNT] = {
+        {"NULL",       ACL3_NULL,      acl3svc_null,   NULL,   0},
+        {"GETACL",     ACL3_GETACL,    acl3svc_getacl, NULL,   0},
+        {"SETACL",     ACL3_SETACL,    acl3svc_setacl, NULL,   0},
+};
+
+rpcsvc_program_t        acl3prog = {
+        .progname       = "ACL3",
+        .prognum        = ACL_PROGRAM,
+        .progver        = ACLV3_VERSION,
+        .progport       = GF_NFS3_PORT,
+        .actors         = acl3svc_actors,
+        .numactors      = ACL3_PROC_COUNT,
+        .min_auth       = AUTH_NULL,
+};
+
+rpcsvc_program_t *
+acl3svc_init(xlator_t *nfsx)
+{
+        struct nfs3_state *ns = NULL;
+        struct nfs_state *nfs = NULL;
+        dict_t *options = NULL;
+        int ret = -1;
+        char *portstr = NULL;
+        static gf_boolean_t acl3_inited = _gf_false;
+
+        /* Already inited */
+        if (acl3_inited)
+                return &acl3prog;
+
+        nfs = (struct nfs_state*)nfsx->private;
+
+        ns = nfs->nfs3state;
+        if (!ns) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "ACL3 init failed");
+                goto err;
+        }
+        acl3prog.private = ns;
+
+        options = dict_new ();
+
+        ret = gf_asprintf (&portstr, "%d", GF_ACL3_PORT);
+        if (ret == -1)
+                goto err;
+
+        ret = dict_set_dynstr (options, "transport.socket.listen-port",
+                               portstr);
+        if (ret == -1)
+                goto err;
+        ret = dict_set_str (options, "transport-type", "socket");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        if (nfs->allow_insecure) {
+                ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
+                if (ret == -1) {
+                        gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                        goto err;
+                }
+                ret = dict_set_str (options, "rpc-auth.ports.insecure", "on");
+                if (ret == -1) {
+                        gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                        goto err;
+                }
+        }
+
+        ret = dict_set_str (options, "transport.address-family", "inet");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, "ACL");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Unable to create listeners");
+                dict_unref (options);
+                goto err;
+        }
+
+        acl3_inited = _gf_true;
+        return &acl3prog;
+err:
+        return NULL;
+}
+
+static int
+acl3_nfs_acl_to_xattr (aclentry *ace,      /* ACL entries to be read */
+                       void     *xattrbuf, /* XATTR buf to be populated */
+                       int      aclcount,  /* No of ACLs to be read */
+                       int      defacl)    /* 1 if DEFAULT ACL */
+{
+        int                       idx      = 0;
+        posix_acl_xattr_header    *xheader = NULL;
+        posix_acl_xattr_entry     *xentry  = NULL;
+
+        if ((!ace) || (!xattrbuf))
+                return (-EINVAL);
+
+        /* ACL count is ZERO, nothing to do */
+        if (!aclcount)
+                return (0);
+
+        if ((aclcount < 0) || (aclcount > NFS_ACL_MAX_ENTRIES))
+                return (-EINVAL);
+
+        xheader = (posix_acl_xattr_header *) (xattrbuf);
+        xentry  = (posix_acl_xattr_entry  *) (xheader + 1);
+
+        /*
+         * For "default ACL", NFSv3 handles the 'type' differently
+         * i.e. by logical OR'ing 'type' with NFS_ACL_DEFAULT.
+         * Which the backend File system does not understand and
+         * that needs to be masked OFF.
+         */
+        xheader->version = POSIX_ACL_XATTR_VERSION;
+
+        for (idx = 0; idx < aclcount; idx++) {
+                xentry->tag  = ace->type;
+                if (defacl)
+                        xentry->tag &= ~NFS_ACL_DEFAULT;
+                xentry->perm = ace->perm;
+
+                switch (xentry->tag) {
+                case POSIX_ACL_USER:
+                case POSIX_ACL_GROUP:
+                        if (xentry->perm & ~S_IRWXO)
+                                return (-EINVAL);
+                        xentry->id = ace->uid;
+                        break;
+                case POSIX_ACL_USER_OBJ:
+                case POSIX_ACL_GROUP_OBJ:
+                case POSIX_ACL_OTHER:
+                        if (xentry->perm & ~S_IRWXO)
+                                return (-EINVAL);
+                        xentry->id = POSIX_ACL_UNDEFINED_ID;
+                        break;
+                case POSIX_ACL_MASK:
+                        /* Solaris sometimes sets additional bits in
+                         * the mask.
+                         */
+                        xentry->perm &= S_IRWXO;
+                        xentry->id = POSIX_ACL_UNDEFINED_ID;
+                        break;
+                default:
+                        return (-EINVAL);
+                }
+
+                xentry++;
+                ace++;
+        }
+
+        /* SUCCESS */
+        return (0);
+}
+
+static int
+acl3_nfs_acl_from_xattr (aclentry  *ace,      /* ACL entries to be filled */
+                         void      *xattrbuf, /* XATTR buf to be read */
+                         int       bufsize,   /* Size of XATTR buffer */
+                         int       defacl)    /*  1 if DEFAULT ACL */
+{
+        int                       idx       = 0;
+        ssize_t                   aclcount  = 0;
+        posix_acl_xattr_header    *xheader  = NULL;
+        posix_acl_xattr_entry     *xentry   = NULL;
+
+        if ((!xattrbuf) || (!ace))
+                return (-EINVAL);
+
+        aclcount = posix_acl_xattr_count (bufsize);
+        if ((aclcount < 0) || (aclcount > NFS_ACL_MAX_ENTRIES))
+                return (-EINVAL);
+
+        xheader = (posix_acl_xattr_header *) (xattrbuf);
+        xentry  = (posix_acl_xattr_entry  *) (xheader + 1);
+
+        /* Check for supported POSIX ACL xattr version */
+        if (xheader->version != POSIX_ACL_XATTR_VERSION)
+                return (-ENOSYS);
+
+        for (idx = 0; idx < (int)aclcount; idx++) {
+                ace->type = xentry->tag;
+                if (defacl) {
+                        /*
+                         * SET the NFS_ACL_DEFAULT flag for default
+                         * ACL which was masked OFF during setfacl().
+                         */
+                        ace->type |= NFS_ACL_DEFAULT;
+                }
+                ace->perm = (xentry->perm & S_IRWXO);
+
+                switch (xentry->tag) {
+                case POSIX_ACL_USER:
+                case POSIX_ACL_GROUP:
+                        ace->uid = xentry->id;
+                        break;
+                case POSIX_ACL_USER_OBJ:
+                case POSIX_ACL_GROUP_OBJ:
+                case POSIX_ACL_MASK:
+                case POSIX_ACL_OTHER:
+                        ace->uid = POSIX_ACL_UNDEFINED_ID;
+                        break;
+                default:
+                        return (-EINVAL);
+                }
+
+
+                xentry++;
+                ace++;
+        }
+
+        /* SUCCESS: ACL count */
+        return aclcount;
+}
diff --git a/xlators/nfs/server/src/acl3.h b/xlators/nfs/server/src/acl3.h
new file mode 100644
index 000000000..03d626f3e
--- /dev/null
+++ b/xlators/nfs/server/src/acl3.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#ifndef _ACL3_H
+#define _ACL3_H
+
+#include "glusterfs-acl.h"
+
+#define GF_ACL3_PORT            38469
+#define GF_ACL                  GF_NFS"-ACL"
+
+/* Flags for the getacl/setacl mode */
+#define NFS_ACL                 0x0001
+#define NFS_ACLCNT              0x0002
+#define NFS_DFACL               0x0004
+#define NFS_DFACLCNT            0x0008
+
+/*
+ * NFSv3, identifies the default ACL by NFS_ACL_DEFAULT. Gluster
+ * NFS needs to mask it OFF before sending it upto POSIX layer
+ * or File system layer.
+ */
+#define NFS_ACL_DEFAULT             0x1000
+
+#define NFS_ACL_MAX_ENTRIES         1024
+
+rpcsvc_program_t *
+acl3svc_init(xlator_t *nfsx);
+
+#endif
diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c
index cebdf5270..47ff3845e 100644
--- a/xlators/nfs/server/src/mount3.c
+++ b/xlators/nfs/server/src/mount3.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -39,23 +30,71 @@
 #include "nfs-mem-types.h"
 #include "nfs.h"
 #include "common-utils.h"
-
+#include "store.h"
 
 #include <errno.h>
 #include <sys/socket.h>
 #include <sys/uio.h>
 
+
+#define IPv4_ADDR_SIZE  32
+
+/* Macro to typecast the parameter to struct sockaddr_in
+ */
+#define SA(addr) ((struct sockaddr_in*)(addr))
+
+/* Macro will mask the ip address with netmask.
+ */
+#define MASKED_IP(ipv4addr, netmask)                    \
+                (ntohl(SA(ipv4addr)->sin_addr.s_addr) & (netmask))
+
+/* Macro will compare two IP address after applying the mask
+ */
+#define COMPARE_IPv4_ADDRS(ip1, ip2, netmask)           \
+                ((MASKED_IP(ip1, netmask)) == (MASKED_IP(ip2, netmask)))
+
+/* This macro will assist in freeing up entire link list
+ * of host_auth_spec structure.
+ */
+#define FREE_HOSTSPEC(exp) do {                                 \
+                struct host_auth_spec *host= exp->hostspec;     \
+                while (NULL != host){                           \
+                        struct host_auth_spec* temp = host;     \
+                        host = host->next;                      \
+                        if (NULL != temp->host_addr) {          \
+                                GF_FREE (temp->host_addr);      \
+                        }                                       \
+                        GF_FREE (temp);                         \
+                }                                               \
+                exp->hostspec = NULL;                           \
+        } while (0)
+
 typedef ssize_t (*mnt3_serializer) (struct iovec outmsg, void *args);
 
+extern void *
+mount3udp_thread (void *argv);
+
+static inline void
+mnt3_export_free (struct mnt3_export *exp)
+{
+        if (!exp)
+                return;
+
+        if (exp->exptype == MNT3_EXPTYPE_DIR)
+                FREE_HOSTSPEC (exp);
+        GF_FREE (exp->expname);
+        GF_FREE (exp);
+}
 
 /* Generic reply function for MOUNTv3 specific replies. */
 int
 mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
 {
-        struct iovec            outmsg = {0, };
-        struct iobuf            *iob = NULL;
-        struct mount3_state     *ms = NULL;
-        int                     ret = -1;
+        struct iovec            outmsg  = {0, };
+        struct iobuf            *iob    = NULL;
+        struct mount3_state     *ms     = NULL;
+        int                     ret     = -1;
+        ssize_t                 msglen  = 0;
         struct iobref           *iobref = NULL;
 
         if (!req)
@@ -81,7 +120,12 @@ mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
         /* Use the given serializer to translate the give C structure in arg
          * to XDR format which will be written into the buffer in outmsg.
          */
-        outmsg.iov_len = sfunc (outmsg, arg);
+        msglen = sfunc (outmsg, arg);
+        if (msglen < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+        outmsg.iov_len = msglen;
 
         iobref = iobref_new ();
         if (iobref == NULL) {
@@ -89,12 +133,14 @@ mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
                 goto ret;
         }
 
-        iobref_add (iobref, iob);
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
 
         /* Then, submit the message for transmission. */
         ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-        iobuf_unref (iob);
-        iobref_unref (iobref);
         if (ret == -1) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Reply submission failed");
                 goto ret;
@@ -102,6 +148,11 @@ mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
 
         ret = 0;
 ret:
+        if (NULL != iob)
+                iobuf_unref (iob);
+        if (NULL != iobref)
+                iobref_unref (iobref);
+
         return ret;
 }
 
@@ -185,13 +236,278 @@ mnt3svc_set_mountres3 (mountstat3 stat, struct nfs3_fh *fh, int *authflavor,
         return res;
 }
 
+/* Read the rmtab from the store_handle and append (or not) the entries to the
+ * mountlist.
+ *
+ * Requires the store_handle to be locked.
+ */
+static int
+__mount_read_rmtab (gf_store_handle_t *sh, struct list_head *mountlist,
+                    gf_boolean_t append)
+{
+        int                     ret = 0;
+        unsigned int            idx = 0;
+        struct mountentry       *me = NULL, *tmp = NULL;
+                                /* me->hostname is a char[MNTPATHLEN] */
+        char                    key[MNTPATHLEN + 11];
+
+        GF_ASSERT (sh && mountlist);
+
+        if (!gf_store_locked_local (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not reading unlocked %s",
+                        sh->path);
+                return -1;
+        }
+
+        if (!append) {
+                list_for_each_entry_safe (me, tmp, mountlist, mlist) {
+                        list_del (&me->mlist);
+                        GF_FREE (me);
+                }
+                me = NULL;
+        }
+
+        for (;;) {
+                char *value = NULL;
+
+                if (me && append) {
+                        /* do not add duplicates */
+                        list_for_each_entry (tmp, mountlist, mlist) {
+                                if (!strcmp(tmp->hostname, me->hostname) &&
+                                    !strcmp(tmp->exname, me->exname)) {
+                                        GF_FREE (me);
+                                        goto dont_add;
+                                }
+                        }
+                        list_add_tail (&me->mlist, mountlist);
+                } else if (me) {
+                        list_add_tail (&me->mlist, mountlist);
+                }
+
+dont_add:
+                me = GF_CALLOC (1, sizeof (*me), gf_nfs_mt_mountentry);
+                if (!me) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory");
+                        ret = -1;
+                        goto out;
+                }
+
+                INIT_LIST_HEAD (&me->mlist);
+
+                snprintf (key, 9 + MNTPATHLEN, "hostname-%d", idx);
+                ret = gf_store_retrieve_value (sh, key, &value);
+                if (ret)
+                        break;
+                strncpy (me->hostname, value, MNTPATHLEN);
+                GF_FREE (value);
+
+                snprintf (key, 11 + MNTPATHLEN, "mountpoint-%d", idx);
+                ret = gf_store_retrieve_value (sh, key, &value);
+                if (ret)
+                        break;
+                strncpy (me->exname, value, MNTPATHLEN);
+                GF_FREE (value);
+
+                idx++;
+                gf_log (GF_MNT, GF_LOG_TRACE, "Read entries %s:%s", me->hostname, me->exname);
+        }
+        gf_log (GF_MNT, GF_LOG_DEBUG, "Read %d entries from '%s'", idx, sh->path);
+        GF_FREE (me);
+out:
+        return ret;
+}
+
+/* Overwrite the contents of the rwtab with te in-memory client list.
+ * Fail gracefully if the stora_handle is not locked.
+ */
+static void
+__mount_rewrite_rmtab(struct mount3_state *ms, gf_store_handle_t *sh)
+{
+        struct mountentry       *me = NULL;
+        char                    key[16];
+        int                     fd, ret;
+        unsigned int            idx = 0;
+
+        if (!gf_store_locked_local (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not modifying unlocked %s",
+                        sh->path);
+                return;
+        }
+
+        fd = gf_store_mkstemp (sh);
+        if (fd == -1) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to open %s", sh->path);
+                return;
+        }
+
+        list_for_each_entry (me, &ms->mountlist, mlist) {
+                snprintf (key, 16, "hostname-%d", idx);
+                ret = gf_store_save_value (fd, key, me->hostname);
+                if (ret)
+                        goto fail;
+
+                snprintf (key, 16, "mountpoint-%d", idx);
+                ret = gf_store_save_value (fd, key, me->exname);
+                if (ret)
+                        goto fail;
+
+                idx++;
+        }
+
+        gf_log (GF_MNT, GF_LOG_DEBUG, "Updated rmtab with %d entries", idx);
+
+        close (fd);
+        if (gf_store_rename_tmppath (sh))
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to overwrite rwtab %s",
+                        sh->path);
+
+        return;
+
+fail:
+        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to update %s", sh->path);
+        close (fd);
+        gf_store_unlink_tmppath (sh);
+}
+
+/* Read the rmtab into a clean ms->mountlist.
+ */
+static void
+mount_read_rmtab (struct mount3_state *ms)
+{
+        gf_store_handle_t       *sh = NULL;
+        struct nfs_state        *nfs = NULL;
+        int                     ret;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return;
+        }
+
+        if (gf_store_lock (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to lock '%s'",
+                        nfs->rmtab);
+                goto out;
+        }
+
+        __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+        gf_store_unlock (sh);
+
+out:
+        gf_store_handle_destroy (sh);
+}
+
+/* Write the ms->mountlist to the rmtab.
+ *
+ * The rmtab could be empty, or it can exists and have been updated by a
+ * different storage server without our knowing.
+ *
+ * 1. takes the store_handle lock on the current rmtab
+ *    - blocks if an other storage server rewrites the rmtab at the same time
+ * 2. [if new_rmtab] takes the store_handle lock on the new rmtab
+ * 3. reads/merges the entries from the current rmtab
+ * 4. [if new_rmtab] reads/merges the entries from the new rmtab
+ * 5. [if new_rmtab] writes the new rmtab
+ * 6. [if not new_rmtab] writes the current rmtab
+ * 7  [if new_rmtab] replaces nfs->rmtab to point to the new location
+ * 8. [if new_rmtab] releases the store_handle lock of the new rmtab
+ * 9. releases the store_handle lock of the old rmtab
+ */
+void
+mount_rewrite_rmtab (struct mount3_state *ms, char *new_rmtab)
+{
+        gf_store_handle_t       *sh = NULL, *nsh = NULL;
+        struct nfs_state        *nfs = NULL;
+        int                     ret;
+        char                    *rmtab = NULL;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return;
+        }
+
+        if (gf_store_lock (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not rewriting '%s'",
+                        nfs->rmtab);
+                goto free_sh;
+        }
+
+        if (new_rmtab) {
+                ret = gf_store_handle_new (new_rmtab, &nsh);
+                if (ret) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                                new_rmtab);
+                        goto unlock_sh;
+                }
+
+                if (gf_store_lock (nsh)) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Not rewriting '%s'",
+                                new_rmtab);
+                        goto free_nsh;
+                }
+        }
+
+        /* always read the currently used rmtab */
+        __mount_read_rmtab (sh, &ms->mountlist, _gf_true);
+
+        if (new_rmtab) {
+                /* read the new rmtab and write changes to the new location */
+                __mount_read_rmtab (nsh, &ms->mountlist, _gf_true);
+                __mount_rewrite_rmtab (ms, nsh);
 
+                /* replace the nfs->rmtab reference to the new rmtab */
+                rmtab = gf_strdup(new_rmtab);
+                if (rmtab == NULL) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory, keeping "
+                                "%s as rmtab", nfs->rmtab);
+                } else {
+                        GF_FREE (nfs->rmtab);
+                        nfs->rmtab = rmtab;
+                }
+
+                gf_store_unlock (nsh);
+        } else {
+                /* rewrite the current (unchanged location) rmtab */
+                __mount_rewrite_rmtab (ms, sh);
+        }
+
+free_nsh:
+        if (new_rmtab)
+                gf_store_handle_destroy (nsh);
+unlock_sh:
+        gf_store_unlock (sh);
+free_sh:
+        gf_store_handle_destroy (sh);
+}
+
+/* Add a new NFS-client to the ms->mountlist and update the rmtab if we can.
+ *
+ * A NFS-client will only be removed from the ms->mountlist in case the
+ * NFS-client sends a unmount request. It is possible that a NFS-client
+ * crashed/rebooted had network loss or something else prevented the NFS-client
+ * to unmount cleanly. In this case, a duplicate entry would be added to the
+ * ms->mountlist, which is wrong and we should prevent.
+ *
+ * It is fully acceptible that the ms->mountlist is not 100% correct, this is a
+ * common issue for all(?) NFS-servers.
+ */
 int
 mnt3svc_update_mountlist (struct mount3_state *ms, rpcsvc_request_t *req,
                           char  *expname)
 {
         struct mountentry       *me = NULL;
+        struct mountentry       *cur = NULL;
         int                     ret = -1;
+        char                    *colon = NULL;
+        struct nfs_state        *nfs = NULL;
+        gf_store_handle_t       *sh = NULL;
 
         if ((!ms) || (!req) || (!expname))
                 return -1;
@@ -201,21 +517,62 @@ mnt3svc_update_mountlist (struct mount3_state *ms, rpcsvc_request_t *req,
         if (!me)
                 return -1;
 
-        strcpy (me->exname, expname);
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                goto free_err;
+        }
+
+        strncpy (me->exname, expname, MNTPATHLEN);
+
         INIT_LIST_HEAD (&me->mlist);
         /* Must get the IP or hostname of the client so we
          * can map it into the mount entry.
          */
         ret = rpcsvc_transport_peername (req->trans, me->hostname, MNTPATHLEN);
         if (ret == -1)
-                goto free_err;
+                goto free_err2;
 
+        colon = strrchr (me->hostname, ':');
+        if (colon) {
+                *colon = '\0';
+        }
         LOCK (&ms->mountlock);
         {
+                /* in case locking fails, we just don't write the rmtab */
+                if (gf_store_lock (sh)) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Failed to lock '%s'"
+                                ", changes will not be written", nfs->rmtab);
+                } else {
+                        __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+                }
+
+                /* do not add duplicates */
+                list_for_each_entry (cur, &ms->mountlist, mlist) {
+                        if (!strcmp(cur->hostname, me->hostname) &&
+                            !strcmp(cur->exname, me->exname)) {
+                                GF_FREE (me);
+                                goto dont_add;
+                        }
+                }
                 list_add_tail (&me->mlist, &ms->mountlist);
+
+                /* only write the rmtab in case it was locked */
+                if (gf_store_locked_local (sh))
+                        __mount_rewrite_rmtab (ms, sh);
         }
+dont_add:
+        if (gf_store_locked_local (sh))
+                gf_store_unlock (sh);
+
         UNLOCK (&ms->mountlock);
 
+free_err2:
+        gf_store_handle_destroy (sh);
+
 free_err:
         if (ret == -1)
                 GF_FREE (me);
@@ -234,6 +591,7 @@ __mnt3_get_volume_id (struct mount3_state *ms, xlator_t *mntxl,
         if ((!ms) || (!mntxl))
                 return ret;
 
+        LOCK (&ms->mountlock);
         list_for_each_entry (exp, &ms->exportlist, explist) {
                 if (exp->vol == mntxl) {
                         uuid_copy (volumeid, exp->volumeid);
@@ -243,6 +601,7 @@ __mnt3_get_volume_id (struct mount3_state *ms, xlator_t *mntxl,
         }
 
 out:
+        UNLOCK (&ms->mountlock);
         return ret;
 }
 
@@ -263,7 +622,7 @@ mnt3svc_lookup_mount_cbk (call_frame_t *frame, void  *cookie,
         rpcsvc_t                *svc = NULL;
         xlator_t                *mntxl = NULL;
         uuid_t                  volumeid = {0, };
-        char                    fhstr[1024];
+        char                    fhstr[1024], *path = NULL;
 
         req = (rpcsvc_request_t *)frame->local;
 
@@ -285,7 +644,15 @@ mnt3svc_lookup_mount_cbk (call_frame_t *frame, void  *cookie,
         if (status != MNT3_OK)
                 goto xmit_res;
 
-        mnt3svc_update_mountlist (ms, req, mntxl->name);
+        path = GF_CALLOC (PATH_MAX, sizeof (char), gf_nfs_mt_char);
+        if (!path) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory");
+                goto xmit_res;
+        }
+
+        snprintf (path, PATH_MAX, "/%s", mntxl->name);
+        mnt3svc_update_mountlist (ms, req, path);
+        GF_FREE (path);
         if (gf_nfs_dvm_off (nfs_state (ms->nfsx))) {
                 fh = nfs3_fh_build_indexed_root_fh (ms->nfsx->children, mntxl);
                 goto xmit_res;
@@ -295,7 +662,7 @@ mnt3svc_lookup_mount_cbk (call_frame_t *frame, void  *cookie,
         fh = nfs3_fh_build_uuid_root_fh (volumeid);
 
 xmit_res:
-        nfs3_fh_to_str (&fh, fhstr);
+        nfs3_fh_to_str (&fh, fhstr, sizeof (fhstr));
         gf_log (GF_MNT, GF_LOG_DEBUG, "MNT reply: fh %s, status: %d", fhstr,
                 status);
         if (op_ret == 0) {
@@ -316,7 +683,7 @@ int
 mnt3_match_dirpath_export (char *expname, char *dirpath)
 {
         int     ret = 0;
-        int     dlen = 0;
+        size_t  dlen;
 
         if ((!expname) || (!dirpath))
                 return 0;
@@ -327,7 +694,7 @@ mnt3_match_dirpath_export (char *expname, char *dirpath)
          * compare.
          */
         dlen = strlen (dirpath);
-        if (dirpath [dlen - 1] == '/')
+        if (dlen && dirpath [dlen - 1] == '/')
                 dirpath [dlen - 1] = '\0';
 
         if (dirpath[0] != '/')
@@ -459,8 +826,8 @@ mnt3_resolve_state_wipe (mnt3_resolve_t *mres)
 /* Sets up the component argument to contain the next component in the path and
  * sets up path as an absolute path starting from the next component.
  */
-char *
-__setup_next_component (char *path, char *component)
+static char *
+setup_next_component (char *path, size_t plen, char *component, size_t clen)
 {
         char    *comp = NULL;
         char    *nextcomp = NULL;
@@ -468,7 +835,7 @@ __setup_next_component (char *path, char *component)
         if ((!path) || (!component))
                 return NULL;
 
-        strcpy (component, path);
+        strncpy (component, path, clen);
         comp = index (component, (int)'/');
         if (!comp)
                 goto err;
@@ -476,7 +843,7 @@ __setup_next_component (char *path, char *component)
         comp++;
         nextcomp = index (comp, (int)'/');
         if (nextcomp) {
-                strcpy (path, nextcomp);
+                strncpy (path, nextcomp, plen);
                 *nextcomp = '\0';
         } else
                 path[0] = '\0';
@@ -506,7 +873,9 @@ __mnt3_resolve_export_subdir_comp (mnt3_resolve_t *mres)
         if (!mres)
                 return ret;
 
-        nextcomp = __setup_next_component (mres->remainingdir, dupsubdir);
+        nextcomp = setup_next_component (mres->remainingdir,
+                                         sizeof (mres->remainingdir),
+                                         dupsubdir, sizeof (dupsubdir));
         if (!nextcomp)
                 goto err;
 
@@ -518,7 +887,7 @@ __mnt3_resolve_export_subdir_comp (mnt3_resolve_t *mres)
         if ((ret < 0) && (ret != -2)) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve and create "
                         "inode: parent gfid %s, entry %s",
-                        uuid_utoa (mres->resolveloc.inode->gfid), nextcomp);
+                        uuid_utoa (gfid), nextcomp);
                 ret = -EFAULT;
                 goto err;
         }
@@ -546,6 +915,7 @@ mnt3_resolve_subdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         rpcsvc_t                *svc = NULL;
         mountres3               res = {0, };
         xlator_t                *mntxl = NULL;
+        char                    *path = NULL;
 
         mres = frame->local;
         mntxl = (xlator_t *)cookie;
@@ -563,15 +933,23 @@ mnt3_resolve_subdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (strlen (mres->remainingdir) <= 0) {
                 op_ret = -1;
                 mntstat = MNT3_OK;
+                path = GF_CALLOC (PATH_MAX, sizeof (char), gf_nfs_mt_char);
+                if (!path) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation "
+                                "failed");
+                        goto err;
+                }
+                snprintf (path, PATH_MAX, "/%s%s", mres->exp->vol->name,
+                         mres->resolveloc.path);
                 mnt3svc_update_mountlist (mres->mstate, mres->req,
-                                          mres->exp->expname);
-                goto err;
+                                          path);
+                GF_FREE (path);
+        } else {
+                mres->parentfh = fh;
+                op_ret = __mnt3_resolve_export_subdir_comp (mres);
+                if (op_ret < 0)
+                        mntstat = mnt3svc_errno_to_mnterr (-op_ret);
         }
-
-        mres->parentfh = fh;
-        op_ret = __mnt3_resolve_export_subdir_comp (mres);
-        if (op_ret < 0)
-                mntstat = mnt3svc_errno_to_mnterr (-op_ret);
 err:
         if (op_ret == -1) {
                 gf_log (GF_MNT, GF_LOG_DEBUG, "Mount reply status: %d",
@@ -614,7 +992,9 @@ __mnt3_resolve_subdir (mnt3_resolve_t *mres)
         if (!mres)
                 return ret;
 
-        firstcomp = __setup_next_component (mres->remainingdir, dupsubdir);
+        firstcomp = setup_next_component (mres->remainingdir,
+                                          sizeof (mres->remainingdir),
+                                          dupsubdir, sizeof (dupsubdir));
         if (!firstcomp)
                 goto err;
 
@@ -637,6 +1017,132 @@ err:
 }
 
 
+/**
+ * This function will verify if the client is allowed to mount
+ * the directory or not. Client's IP address will be compared with
+ * allowed IP list or range present in mnt3_export structure.
+ *
+ * @param req - RPC request. This structure contains client's IP address.
+ * @param export - mnt3_export structure. Contains allowed IP list/range.
+ *
+ * @return 0 - on Success and -EACCES on failure.
+ */
+int
+mnt3_verify_auth (rpcsvc_request_t *req, struct mnt3_export *export)
+{
+        int                     retvalue = -EACCES;
+        int                     ret = 0;
+        int                     shiftbits = 0;
+        uint32_t                ipv4netmask = 0;
+        uint32_t                routingprefix = 0;
+        struct host_auth_spec   *host = NULL;
+        struct sockaddr_in      *client_addr = NULL;
+        struct sockaddr_in      *allowed_addr = NULL;
+        struct addrinfo         *allowed_addrinfo = NULL;
+
+        /* Sanity check */
+        if ((NULL == req) ||
+            (NULL == req->trans) ||
+            (NULL == export) ||
+            (NULL == export->hostspec)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Invalid argument");
+                return retvalue;
+        }
+
+        host = export->hostspec;
+
+
+        /* Client's IP address. */
+        client_addr = (struct sockaddr_in *)(&(req->trans->peerinfo.sockaddr));
+
+        /* Try to see if the client IP matches the allowed IP list.*/
+        while (NULL != host){
+                GF_ASSERT (host->host_addr);
+
+                if (NULL != allowed_addrinfo) {
+                        freeaddrinfo (allowed_addrinfo);
+                        allowed_addrinfo = NULL;
+                }
+
+                /* Get the addrinfo for the allowed host (host_addr). */
+                ret = getaddrinfo (host->host_addr,
+                                NULL,
+                                NULL,
+                                &allowed_addrinfo);
+                if (0 != ret){
+                        gf_log (GF_MNT, GF_LOG_ERROR, "getaddrinfo: %s\n",
+                                gai_strerror (ret));
+                        host = host->next;
+
+                        /* Failed to get IP addrinfo. Continue to check other
+                         * allowed IPs in the list.
+                         */
+                        continue;
+                }
+
+                allowed_addr = (struct sockaddr_in *)(allowed_addrinfo->ai_addr);
+
+                if (NULL == allowed_addr) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Invalid structure");
+                        break;
+                }
+
+                if (AF_INET == allowed_addr->sin_family){
+                        if (IPv4_ADDR_SIZE < host->routeprefix) {
+                                gf_log (GF_MNT, GF_LOG_ERROR, "invalid IP "
+                                        "configured for export-dir AUTH");
+                                host = host->next;
+                                continue;
+                        }
+
+                        /* -1 means no route prefix is provided. In this case
+                         * the IP should be an exact match. Which is same as
+                         * providing a route prefix of IPv4_ADDR_SIZE.
+                         */
+                        if (-1 == host->routeprefix) {
+                                routingprefix = IPv4_ADDR_SIZE;
+                        } else {
+                                routingprefix = host->routeprefix;
+                        }
+
+                        /* Create a mask from the routing prefix. User provided
+                         * CIDR address is split into IP address (host_addr) and
+                         * routing prefix (routeprefix). This CIDR address may
+                         * denote a single, distinct interface address or the
+                         * beginning address of an entire network.
+                         *
+                         * e.g. the IPv4 block 192.168.100.0/24 represents the
+                         * 256 IPv4 addresses from 192.168.100.0 to
+                         * 192.168.100.255.
+                         * Therefore to check if an IP matches 192.168.100.0/24
+                         * we should mask the IP with FFFFFF00 and compare it
+                         * with host address part of CIDR.
+                         */
+                        shiftbits = IPv4_ADDR_SIZE - routingprefix;
+                        ipv4netmask = 0xFFFFFFFFUL << shiftbits;
+
+                        /* Mask both the IPs and then check if they match
+                         * or not. */
+                        if (COMPARE_IPv4_ADDRS (allowed_addr,
+                                                client_addr,
+                                                ipv4netmask)){
+                                retvalue = 0;
+                                break;
+                        }
+                }
+
+                /* Client IP didn't match the allowed IP.
+                 * Check with the next allowed IP.*/
+               host = host->next;
+        }
+
+        if (NULL != allowed_addrinfo) {
+               freeaddrinfo (allowed_addrinfo);
+        }
+
+        return retvalue;
+}
+
 int
 mnt3_resolve_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
                      struct mnt3_export *exp, char *subdir)
@@ -648,6 +1154,16 @@ mnt3_resolve_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
         if ((!req) || (!ms) || (!exp) || (!subdir))
                 return ret;
 
+        /* Need to check AUTH */
+        if (NULL != exp->hostspec) {
+                ret = mnt3_verify_auth (req, exp);
+                if (0 != ret) {
+                        gf_log (GF_MNT,GF_LOG_ERROR,
+                                        "AUTH verification failed");
+                        return ret;
+                }
+        }
+
         mres = GF_CALLOC (1, sizeof (mnt3_resolve_t), gf_nfs_mt_mnt3_resolve);
         if (!mres) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
@@ -657,7 +1173,7 @@ mnt3_resolve_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
         mres->exp = exp;
         mres->mstate = ms;
         mres->req = req;
-        strcpy (mres->remainingdir, subdir);
+        strncpy (mres->remainingdir, subdir, MNTPATHLEN);
         if (gf_nfs_dvm_off (nfs_state (ms->nfsx)))
                 pfh = nfs3_fh_build_indexed_root_fh (mres->mstate->nfsx->children, mres->exp->vol);
         else
@@ -732,6 +1248,7 @@ mnt3_mntpath_to_export (struct mount3_state *ms, char *dirpath)
         if ((!ms) || (!dirpath))
                 return NULL;
 
+        LOCK (&ms->mountlock);
         list_for_each_entry (exp, &ms->exportlist, explist) {
 
                 /* Search for the an exact match with the volume */
@@ -745,6 +1262,7 @@ mnt3_mntpath_to_export (struct mount3_state *ms, char *dirpath)
 
         gf_log (GF_MNT, GF_LOG_DEBUG, "Export not found");
 foundexp:
+        UNLOCK (&ms->mountlock);
         return found;
 }
 
@@ -754,24 +1272,36 @@ mnt3_check_client_net (struct mount3_state *ms, rpcsvc_request_t *req,
                        xlator_t *targetxl)
 {
 
-        rpcsvc_t        *svc = NULL;
-        int             ret = -1;
+        rpcsvc_t                *svc = NULL;
+        rpc_transport_t         *trans = NULL;
+        struct sockaddr_storage sastorage = {0,};
+        char                    peer[RPCSVC_PEER_STRLEN] = {0,};
+        int                     ret = -1;
 
         if ((!ms) || (!req) || (!targetxl))
                 return -1;
 
         svc = rpcsvc_request_service (req);
-        ret = rpcsvc_transport_peer_check (svc->options, targetxl->name,
-                                           rpcsvc_request_transport (req));
+
+        trans = rpcsvc_request_transport (req);
+        ret = rpcsvc_transport_peeraddr (trans, peer, RPCSVC_PEER_STRLEN,
+                                         &sastorage, sizeof (sastorage));
+        if (ret != 0) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to get peer addr: %s",
+                        gai_strerror (ret));
+        }
+
+        ret = rpcsvc_auth_check (svc, targetxl->name, trans);
         if (ret == RPCSVC_AUTH_REJECT) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Peer not allowed");
+                gf_log (GF_MNT, GF_LOG_INFO, "Peer %s  not allowed", peer);
                 goto err;
         }
 
         ret = rpcsvc_transport_privport_check (svc, targetxl->name,
                                                rpcsvc_request_transport (req));
         if (ret == RPCSVC_AUTH_REJECT) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Unprivileged port not allowed");
+                gf_log (GF_MNT, GF_LOG_INFO, "Peer %s rejected. Unprivileged "
+                        "port not allowed", peer);
                 goto err;
         }
 
@@ -788,7 +1318,8 @@ mnt3_parse_dir_exports (rpcsvc_request_t *req, struct mount3_state *ms,
         char                    volname[1024];
         struct mnt3_export      *exp = NULL;
         char                    *volname_ptr = NULL;
-        int                     ret = -1;
+        int                     ret = -ENOENT;
+        struct nfs_state        *nfs = NULL;
 
         if ((!ms) || (!subdir))
                 return -1;
@@ -802,10 +1333,26 @@ mnt3_parse_dir_exports (rpcsvc_request_t *req, struct mount3_state *ms,
         if (!exp)
                 goto err;
 
+        nfs = (struct nfs_state *)ms->nfsx->private;
+        if (!nfs)
+                goto err;
+
+        if (!nfs_subvolume_started (nfs, exp->vol)) {
+                gf_log (GF_MNT, GF_LOG_DEBUG,
+                        "Volume %s not started", exp->vol->name);
+                goto err;
+        }
+
+        if (mnt3_check_client_net (ms, req, exp->vol) == RPCSVC_AUTH_REJECT) {
+                gf_log (GF_MNT, GF_LOG_DEBUG, "Client mount not allowed");
+                ret = -EACCES;
+                goto err;
+        }
+
         ret = mnt3_resolve_subdir (req, ms, exp, subdir);
         if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve export dir: %s"
-                        , subdir);
+                gf_log (GF_MNT, GF_LOG_ERROR,
+                        "Failed to resolve export dir: %s", subdir);
                 goto err;
         }
 
@@ -845,10 +1392,6 @@ mnt3_find_export (rpcsvc_request_t *req, char *path, struct mnt3_export **e)
         }
 
         ret = mnt3_parse_dir_exports (req, ms, path);
-        if (ret == 0) {
-                ret = -2;
-                goto err;
-        }
 
 err:
         return ret;
@@ -886,17 +1429,26 @@ mnt3svc_mnt (rpcsvc_request_t *req)
                 goto rpcerr;
         }
 
-        ret = 0;
         nfs = (struct nfs_state *)ms->nfsx->private;
         gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s", path);
         ret = mnt3_find_export (req, path, &exp);
-        if (ret == -2) {
-                ret = 0;
-                goto rpcerr;
-        } else if (ret < 0) {
-                ret = -1;
-                mntstat = MNT3ERR_NOENT;
+        if (ret < 0) {
+                mntstat = mnt3svc_errno_to_mnterr (-ret);
                 goto mnterr;
+        } else if (!exp) {
+                /*
+                 * SPECIAL CASE: exp is NULL if "path" is subdir in
+                 * call to mnt3_find_export().
+                 *
+                 * This is subdir mount, we are already DONE!
+                 * nfs_subvolume_started() and mnt3_check_client_net()
+                 * validation are done in mnt3_parse_dir_exports()
+                 * which is invoked through mnt3_find_export().
+                 *
+                 * TODO: All mount should happen thorugh mnt3svc_mount()
+                 *       It needs more clean up.
+                 */
+                return (0);
         }
 
         if (!nfs_subvolume_started (nfs, exp->vol)) {
@@ -956,6 +1508,9 @@ __build_mountlist (struct mount3_state *ms, int *count)
         if ((!ms) || (!count))
                 return NULL;
 
+        /* read rmtab, other peers might have updated it */
+        mount_read_rmtab(ms);
+
         *count = 0;
         gf_log (GF_MNT, GF_LOG_DEBUG, "Building mount list:");
         list_for_each_entry (me, &ms->mountlist, mlist) {
@@ -966,6 +1521,8 @@ __build_mountlist (struct mount3_state *ms, int *count)
                                 " failed");
                         goto free_list;
                 }
+                if (!first)
+                        first = mlist;
 
                 mlist->ml_directory = GF_CALLOC (namelen + 2, sizeof (char),
                                                  gf_nfs_mt_char);
@@ -975,8 +1532,7 @@ __build_mountlist (struct mount3_state *ms, int *count)
                         goto free_list;
                 }
 
-                strcpy (mlist->ml_directory, "/");
-                strcat (mlist->ml_directory, me->exname);
+                strcpy (mlist->ml_directory, me->exname);
 
                 namelen = strlen (me->hostname);
                 mlist->ml_hostname = GF_CALLOC (namelen + 2, sizeof (char),
@@ -997,9 +1553,6 @@ __build_mountlist (struct mount3_state *ms, int *count)
                 } else
                         prev = mlist;
 
-                if (!first)
-                        first = mlist;
-
                 (*count)++;
         }
 
@@ -1076,67 +1629,71 @@ rpcerr:
 
 
 int
-__mnt3svc_umount (struct mount3_state *ms, char *dirpath, char *hostname)
+mnt3svc_umount (struct mount3_state *ms, char *dirpath, char *hostname)
 {
         struct mountentry       *me = NULL;
-        char                    *exname = NULL;
         int                     ret = -1;
+        gf_store_handle_t       *sh = NULL;
+        struct nfs_state        *nfs = NULL;
 
         if ((!ms) || (!dirpath) || (!hostname))
                 return -1;
 
-        if (list_empty (&ms->mountlist))
-                return 0;
-
-        if (dirpath[0] == '/')
-                exname = dirpath+1;
-        else
-                exname = dirpath;
+        nfs = (struct nfs_state *)ms->nfsx->private;
 
-        list_for_each_entry (me, &ms->mountlist, mlist) {
-               if ((strcmp (me->exname, exname) == 0) &&
-                    (strcmp (me->hostname, hostname) == 0)) {
-                       ret = 0;
-                       break;
-               }
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return 0;
         }
 
-        /* Need this check here because at the end of the search me might still
-         * be pointing to the last entry, which may not be the one we're
-         * looking for.
-         */
-        if (ret == -1)  {/* Not found in list. */
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Export not found");
-                goto ret;
+        ret = gf_store_lock (sh);
+        if (ret) {
+                goto out_free;
         }
 
-        if (!me)
-                goto ret;
+        LOCK (&ms->mountlock);
+        {
+                __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+                if (list_empty (&ms->mountlist)) {
+                        ret = 0;
+                        goto out_unlock;
+                }
 
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Unmounting: dir %s, host: %s",
-                me->exname, me->hostname);
-        list_del (&me->mlist);
-        GF_FREE (me);
-        ret = 0;
-ret:
-        return ret;
-}
+                ret = -1;
+                list_for_each_entry (me, &ms->mountlist, mlist) {
+                       if ((strcmp (me->exname, dirpath) == 0) &&
+                            (strcmp (me->hostname, hostname) == 0)) {
+                               ret = 0;
+                               break;
+                       }
+                }
 
+                /* Need this check here because at the end of the search me
+                 * might still be pointing to the last entry, which may not be
+                 * the one we're looking for.
+                 */
+                if (ret == -1)  {/* Not found in list. */
+                        gf_log (GF_MNT, GF_LOG_TRACE, "Export not found");
+                        goto out_unlock;
+                }
 
+                if (!me)
+                        goto out_unlock;
 
-int
-mnt3svc_umount (struct mount3_state *ms, char *dirpath, char *hostname)
-{
-        int ret = -1;
-        if ((!ms) || (!dirpath) || (!hostname))
-                return -1;
+                gf_log (GF_MNT, GF_LOG_DEBUG, "Unmounting: dir %s, host: %s",
+                        me->exname, me->hostname);
 
-        LOCK (&ms->mountlock);
-        {
-               ret = __mnt3svc_umount (ms, dirpath, hostname);
+                list_del (&me->mlist);
+                GF_FREE (me);
+                __mount_rewrite_rmtab (ms, sh);
         }
+out_unlock:
         UNLOCK (&ms->mountlock);
-
+        gf_store_unlock (sh);
+out_free:
+        gf_store_handle_destroy (sh);
         return ret;
 }
 
@@ -1150,6 +1707,7 @@ mnt3svc_umnt (rpcsvc_request_t *req)
         int                     ret = -1;
         struct mount3_state     *ms = NULL;
         mountstat3              mstat = MNT3_OK;
+        char                    *colon = NULL;
 
         if (!req)
                 return -1;
@@ -1176,37 +1734,24 @@ mnt3svc_umnt (rpcsvc_request_t *req)
         if (ret != 0) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get remote name: %s",
                         gai_strerror (ret));
-                goto try_umount_with_addr;
-        }
-
-        gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s, hostname: %s", dirpath,
-                hostname);
-        ret = mnt3svc_umount (ms, dirpath, hostname);
-
-        /* Unmount succeeded with the given hostname. */
-        if (ret == 0)
-                goto snd_reply;
-
-try_umount_with_addr:
-        if (ret != 0)
-                ret = rpcsvc_transport_peeraddr (req->trans, hostname,
-                                                 MNTPATHLEN, NULL, 0);
-
-        if (ret != 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get remote addr: %s",
-                        gai_strerror (ret));
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
                 goto rpcerr;
         }
 
+        colon = strrchr (hostname, ':');
+        if (colon) {
+                *colon= '\0';
+        }
         gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s, hostname: %s", dirpath,
                 hostname);
         ret = mnt3svc_umount (ms, dirpath, hostname);
-        if (ret == -1)
-                mstat = MNT3ERR_INVAL;
 
-        ret = 0;
-snd_reply:
+        if (ret == -1) {
+                ret = 0;
+                mstat = MNT3ERR_NOENT;
+        }
+        /* FIXME: also take care of the corner case where the
+         * client was resolvable at mount but not at the umount - vice-versa.
+         */
         mnt3svc_submit_reply (req, &mstat,
                               (mnt3_serializer)xdr_serialize_mountstat3);
 
@@ -1296,6 +1841,10 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
                 return NULL;
 
         nfs = (struct nfs_state *)ms->nfsx->private;
+        if (!nfs)
+                return NULL;
+
+        LOCK (&ms->mountlock);
         list_for_each_entry(ent, &ms->exportlist, explist) {
 
                 /* If volume is not started yet, do not list it for tools like
@@ -1311,7 +1860,8 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
                                 " failed");
                         goto free_list;
                 }
-
+                 if (!first)
+                         first = elist;
                 elist->ex_dir = GF_CALLOC (namelen + 2, sizeof (char),
                                            gf_nfs_mt_char);
                 if (!elist->ex_dir) {
@@ -1319,16 +1869,10 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
                                 " failed");
                         goto free_list;
                 }
-
                 strcpy (elist->ex_dir, ent->expname);
 
                 addrstr = rpcsvc_volume_allowed (svc->options,
                                                  ent->vol->name);
-                if (addrstr)
-                        addrstr = gf_strdup (addrstr);
-                else
-                        addrstr = gf_strdup ("No Access");
-
                 elist->ex_groups = GF_CALLOC (1, sizeof (struct groupnode),
                                               gf_nfs_mt_groupnode);
                 if (!elist->ex_groups) {
@@ -1336,21 +1880,29 @@ mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
                                 " failed");
                         goto free_list;
                 }
+                /*This check has to be done after checking
+                 * elist->ex_groups allocation check to avoid resource leak;
+                */
+                if (addrstr)
+                        addrstr = gf_strdup (addrstr);
+                else
+                        addrstr = gf_strdup ("No Access");
 
+                if (!addrstr) {
+                        goto free_list;
+                }
                 elist->ex_groups->gr_name = addrstr;
                 if (prev) {
                         prev->ex_next = elist;
                         prev = elist;
                 } else
                         prev = elist;
-
-                if (!first)
-                        first = elist;
         }
 
         ret = 0;
 
 free_list:
+        UNLOCK (&ms->mountlock);
         if (ret == -1) {
                 xdr_free_exports_list (first);
                 first = NULL;
@@ -1400,7 +1952,233 @@ err:
         return ret;
 }
 
+/* just declaring, definition is way down below */
+rpcsvc_program_t        mnt3prog;
+
+/* nfs3_rootfh used by mount3udp thread needs to access mount3prog.private
+ * directly as we don't have nfs xlator pointer to dereference it. But thats OK
+ */
+
+struct nfs3_fh *
+nfs3_rootfh (char* path)
+{
+        struct mount3_state     *ms = NULL;
+        struct nfs3_fh          *fh = NULL;
+        struct mnt3_export      *exp = NULL;
+        inode_t                 *inode = NULL;
+        char                    *tmp = NULL;
+
+        ms = mnt3prog.private;
+        exp = mnt3_mntpath_to_export (ms, path);
+        if (exp == NULL)
+                goto err;
+
+        tmp = (char *)path;
+        tmp = strchr (tmp, '/');
+        if (tmp == NULL)
+                tmp = "/";
+
+        inode = inode_from_path (exp->vol->itable, tmp);
+        if (inode == NULL)
+                goto err;
+
+        fh = GF_CALLOC (1, sizeof(*fh), gf_nfs_mt_nfs3_fh);
+        if (fh == NULL)
+                goto err;
+        nfs3_build_fh (inode, exp->volumeid, fh);
+
+err:
+        if (inode)
+                inode_unref (inode);
+        return fh;
+}
+
+int
+mount3udp_add_mountlist (char *host, dirpath *expname)
+{
+        struct mountentry       *me = NULL;
+        struct mount3_state     *ms = NULL;
+        char                    *export = NULL;
+
+        ms = mnt3prog.private;
+        me = GF_CALLOC (1, sizeof (*me), gf_nfs_mt_mountentry);
+        if (!me)
+                return -1;
+        export = (char *)expname;
+        while (*export == '/')
+                export++;
+
+        strncpy (me->exname, export, MNTPATHLEN);
+        strncpy (me->hostname, host, MNTPATHLEN);
+        INIT_LIST_HEAD (&me->mlist);
+        LOCK (&ms->mountlock);
+        {
+                list_add_tail (&me->mlist, &ms->mountlist);
+                mount_rewrite_rmtab(ms, NULL);
+        }
+        UNLOCK (&ms->mountlock);
+        return 0;
+}
 
+int
+mount3udp_delete_mountlist (char *hostname, dirpath *expname)
+{
+        struct mount3_state     *ms = NULL;
+        char                    *export = NULL;
+
+        ms = mnt3prog.private;
+        export = (char *)expname;
+        while (*export == '/')
+                export++;
+        mnt3svc_umount (ms, export, hostname);
+        return 0;
+}
+
+/**
+ * This function will parse the hostip (IP addres, IP range, or hostname)
+ * and fill the host_auth_spec structure.
+ *
+ * @param hostspec - struct host_auth_spec
+ * @param hostip   - IP address, IP range (CIDR format) or hostname
+ *
+ * @return 0 - on success and -1 on failure
+ */
+int
+mnt3_export_fill_hostspec (struct host_auth_spec* hostspec, const char* hostip)
+{
+        char *ipdupstr = NULL;
+        char *savptr = NULL;
+        char *ip = NULL;
+        char *token = NULL;
+        int  ret = -1;
+
+        /* Create copy of the string so that the source won't change
+         */
+        ipdupstr = gf_strdup (hostip);
+        if (NULL == ipdupstr) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                goto err;
+        }
+
+        ip = strtok_r (ipdupstr, "/", &savptr);
+        hostspec->host_addr = gf_strdup (ip);
+        if (NULL == hostspec->host_addr) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                goto err;
+        }
+
+        /* Check if the IP is in <IP address> / <Range> format.
+         * If yes, then strip the range and store it separately.
+         */
+        token = strtok_r (NULL, "/", &savptr);
+
+        if (NULL == token) {
+              hostspec->routeprefix = -1;
+        } else {
+              hostspec->routeprefix = atoi (token);
+        }
+
+        // success
+        ret = 0;
+err:
+        if (NULL != ipdupstr) {
+                GF_FREE (ipdupstr);
+        }
+        return ret;
+}
+
+
+/**
+ * This function will parse the AUTH parameter passed along with
+ * "export-dir" option. If AUTH parameter is present then it will be
+ * stripped from exportpath and stored in mnt3_export (exp) structure.
+ *
+ * @param exp - mnt3_export structure. Holds information needed for mount.
+ * @param exportpath - Value of "export-dir" key. Holds both export path
+ *                     and AUTH parameter for the path.
+ *                     exportpath format: <abspath>[(hostdesc[|hostspec|...])]
+ *
+ * @return This function will return 0 on success and -1 on failure.
+ */
+int
+mnt3_export_parse_auth_param (struct mnt3_export* exp, char* exportpath)
+{
+        char *token = NULL;
+        char *savPtr = NULL;
+        char *hostip = NULL;
+        struct host_auth_spec *host = NULL;
+        int ret = 0;
+
+        /* Using exportpath directly in strtok_r because we want
+         * to strip off AUTH parameter from exportpath. */
+        token = strtok_r (exportpath, "(", &savPtr);
+
+        /* Get the next token, which will be the AUTH parameter. */
+        token = strtok_r (NULL, ")", &savPtr);
+
+        if (NULL == token) {
+                /* If AUTH is not present then we should return success. */
+                return 0;
+        }
+
+        /* Free any previously allocated hostspec structure. */
+        if (NULL != exp->hostspec) {
+                GF_FREE (exp->hostspec);
+                exp->hostspec = NULL;
+        }
+
+        exp->hostspec = GF_CALLOC (1,
+                        sizeof (*(exp->hostspec)),
+                        gf_nfs_mt_auth_spec);
+        if (NULL == exp->hostspec){
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                return -1;
+        }
+
+        /* AUTH parameter can have multiple entries. For each entry
+         * a host_auth_spec structure is created. */
+        host = exp->hostspec;
+
+        hostip = strtok_r (token, "|", &savPtr);
+
+        /* Parse all AUTH parameters separated by '|' */
+        while (NULL != hostip){
+                ret = mnt3_export_fill_hostspec (host, hostip);
+                if (0 != ret) {
+                        gf_log(GF_MNT, GF_LOG_WARNING,
+                                        "Failed to parse hostspec: %s", hostip);
+                        goto err;
+                }
+
+                hostip = strtok_r (NULL, "|", &savPtr);
+                if (NULL == hostip) {
+                        break;
+                }
+
+                host->next = GF_CALLOC (1, sizeof (*(host)),
+                                gf_nfs_mt_auth_spec);
+                if (NULL == host->next){
+                        gf_log (GF_MNT,GF_LOG_ERROR,
+                                        "Memory allocation failed");
+                        goto err;
+                }
+                host = host->next;
+        }
+
+        /* In case of success return from here */
+        return 0;
+err:
+        /* In case of failure free up hostspec structure.  */
+        FREE_HOSTSPEC (exp);
+
+        return -1;
+}
+
+/**
+ * exportpath will also have AUTH options (ip address, subnet address or
+ * hostname) mentioned.
+ * exportpath format: <abspath>[(hostdesc[|hostspec|...])]
+ */
 struct mnt3_export *
 mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
                       uuid_t volumeid)
@@ -1418,6 +2196,20 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
                 return NULL;
         }
 
+        if (NULL != exportpath) {
+                /* If exportpath is not NULL then we should check if AUTH
+                 * parameter is present or not. If AUTH parameter is present
+                 * then it will be stripped and stored in mnt3_export (exp)
+                 * structure.
+                 */
+                if (0 != mnt3_export_parse_auth_param (exp, exportpath)){
+                        gf_log (GF_MNT, GF_LOG_ERROR,
+                                                "Failed to parse auth param");
+                        goto err;
+                }
+        }
+
+
         INIT_LIST_HEAD (&exp->explist);
         if (exportpath)
                 alloclen = strlen (xl->name) + 2 + strlen (exportpath);
@@ -1427,8 +2219,6 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
         exp->expname = GF_CALLOC (alloclen, sizeof (char), gf_nfs_mt_char);
         if (!exp->expname) {
                 gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                GF_FREE (exp);
-                exp = NULL;
                 goto err;
         }
 
@@ -1445,8 +2235,9 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
                 ret = snprintf (exp->expname, alloclen, "/%s", xl->name);
         }
         if (ret < 0) {
-                gf_log (xl->name, GF_LOG_WARNING,
-                        "failed to get the export name");
+                gf_log (xl->name, GF_LOG_ERROR,
+                        "Failed to set the export name");
+                goto err;
         }
         /* Just copy without discrimination, we'll determine whether to
          * actually use it when a mount request comes in and a file handle
@@ -1454,7 +2245,16 @@ mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
          */
         uuid_copy (exp->volumeid, volumeid);
         exp->vol = xl;
+
+        /* On success we should return from here*/
+        return exp;
 err:
+        /* On failure free exp and it's members.*/
+        if (NULL != exp) {
+                mnt3_export_free (exp);
+                exp = NULL;
+        }
+
         return exp;
 }
 
@@ -1494,8 +2294,7 @@ __mnt3_init_volume_direxports (struct mount3_state *ms, xlator_t *xlator,
 
         ret = 0;
 err:
-        if (dupopt)
-                GF_FREE (dupopt);
+        GF_FREE (dupopt);
 
         return ret;
 }
@@ -1616,8 +2415,11 @@ __mnt3_init_volume_export (struct mount3_state *ms, dict_t *opts)
                 goto err;
         }
 
-        gf_string2boolean (optstr, &boolt);
-        ret = 0;
+        ret = gf_string2boolean (optstr, &boolt);
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to convert"
+                        " string to boolean");
+        }
 
 err:
         if (boolt == _gf_false) {
@@ -1656,8 +2458,11 @@ __mnt3_init_dir_export (struct mount3_state *ms, dict_t *opts)
                 goto err;
         }
 
-        gf_string2boolean (optstr, &boolt);
-        ret = 0;
+        ret = gf_string2boolean (optstr, &boolt);
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to convert"
+                        " string to boolean");
+         }
 
 err:
         if (boolt == _gf_false) {
@@ -1755,18 +2560,20 @@ out:
 }
 
 rpcsvc_actor_t  mnt3svc_actors[MOUNT3_PROC_COUNT] = {
-        {"NULL", MOUNT3_NULL, mnt3svc_null, NULL, NULL, 0},
-        {"MNT", MOUNT3_MNT, mnt3svc_mnt, NULL, NULL, 0},
-        {"DUMP", MOUNT3_DUMP, mnt3svc_dump, NULL, NULL, 0},
-        {"UMNT", MOUNT3_UMNT, mnt3svc_umnt, NULL, NULL, 0},
-        {"UMNTALL", MOUNT3_UMNTALL, mnt3svc_umntall, NULL, NULL, 0},
-        {"EXPORT", MOUNT3_EXPORT, mnt3svc_export, NULL, NULL, 0}
+        {"NULL",    MOUNT3_NULL,    mnt3svc_null,    NULL, 0, DRC_NA},
+        {"MNT",     MOUNT3_MNT,     mnt3svc_mnt,     NULL, 0, DRC_NA},
+        {"DUMP",    MOUNT3_DUMP,    mnt3svc_dump,    NULL, 0, DRC_NA},
+        {"UMNT",    MOUNT3_UMNT,    mnt3svc_umnt,    NULL, 0, DRC_NA},
+        {"UMNTALL", MOUNT3_UMNTALL, mnt3svc_umntall, NULL, 0, DRC_NA},
+        {"EXPORT",  MOUNT3_EXPORT,  mnt3svc_export,  NULL, 0, DRC_NA}
 };
 
 
 
 /* Static init parts are assigned here, dynamic ones are done in
  * mnt3svc_init and mnt3_init_state.
+ * Making MOUNT3 a synctask so that the blocking DNS calls during rpc auth
+ * gets offloaded to syncenv, keeping the main/poll thread unblocked
  */
 rpcsvc_program_t        mnt3prog = {
                         .progname       = "MOUNT3",
@@ -1776,9 +2583,11 @@ rpcsvc_program_t        mnt3prog = {
                         .actors         = mnt3svc_actors,
                         .numactors      = MOUNT3_PROC_COUNT,
                         .min_auth       = AUTH_NULL,
+                        .synctask       = _gf_true,
 };
 
 
+
 rpcsvc_program_t *
 mnt3svc_init (xlator_t *nfsx)
 {
@@ -1787,6 +2596,7 @@ mnt3svc_init (xlator_t *nfsx)
         dict_t                  *options = NULL;
         char                    *portstr = NULL;
         int                      ret = -1;
+        pthread_t                udp_thread;
 
         if (!nfsx || !nfsx->private)
                 return NULL;
@@ -1829,13 +2639,16 @@ mnt3svc_init (xlator_t *nfsx)
                 }
         }
 
-        rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
+        ret= rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
         if (ret == -1) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
                 dict_unref (options);
                 goto err;
         }
 
+        if (nfs->mount_udp) {
+                pthread_create (&udp_thread, NULL, mount3udp_thread, NULL);
+        }
         return &mnt3prog;
 err:
         return NULL;
@@ -1843,12 +2656,12 @@ err:
 
 
 rpcsvc_actor_t  mnt1svc_actors[MOUNT1_PROC_COUNT] = {
-        {"NULL", MOUNT1_NULL, mnt3svc_null, NULL, NULL, 0},
-        {{0, 0}, },
-        {"DUMP", MOUNT1_DUMP, mnt3svc_dump, NULL, NULL, 0},
-        {"UMNT", MOUNT1_UMNT, mnt3svc_umnt, NULL, NULL, 0},
-        {{0, 0}, },
-        {"EXPORT", MOUNT1_EXPORT, mnt3svc_export, NULL, NULL, 0}
+        {"NULL",    MOUNT1_NULL,    mnt3svc_null,   NULL, 0, DRC_NA},
+        {"MNT",     MOUNT1_MNT,     NULL,           NULL, 0, DRC_NA },
+        {"DUMP",    MOUNT1_DUMP,    mnt3svc_dump,   NULL, 0, DRC_NA},
+        {"UMNT",    MOUNT1_UMNT,    mnt3svc_umnt,   NULL, 0, DRC_NA},
+        {"UMNTALL", MOUNT1_UMNTALL, NULL,           NULL, 0, DRC_NA},
+        {"EXPORT",  MOUNT1_EXPORT,  mnt3svc_export, NULL, 0, DRC_NA}
 };
 
 rpcsvc_program_t        mnt1prog = {
@@ -1859,6 +2672,7 @@ rpcsvc_program_t        mnt1prog = {
                         .actors         = mnt1svc_actors,
                         .numactors      = MOUNT1_PROC_COUNT,
                         .min_auth       = AUTH_NULL,
+                        .synctask       = _gf_true,
 };
 
 
@@ -1913,7 +2727,7 @@ mnt1svc_init (xlator_t *nfsx)
                 }
         }
 
-        rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
         if (ret == -1) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
                 dict_unref (options);
@@ -1925,4 +2739,43 @@ err:
         return NULL;
 }
 
+int
+mount_reconfigure_state (xlator_t *nfsx, dict_t *options)
+{
+        int                    ret      = -1;
+        struct nfs_state       *nfs     = NULL;
+        struct mount3_state    *ms      = NULL;
+        struct mnt3_export     *exp     = NULL;
+        struct mnt3_export     *texp  = NULL;
+
+        if ((!nfsx) || (!options))
+                return (-1);
 
+        nfs = (struct nfs_state *)nfs_state (nfsx);
+        if (!nfs)
+                return (-1);
+
+        ms = nfs->mstate;
+        if (!ms)
+                return (-1);
+
+        /*
+         * Free() up the old export list. mnt3_init_options() will
+         * rebuild the export list from scratch. Do it with locking
+         * to avoid unnecessary race conditions.
+         */
+        LOCK (&ms->mountlock);
+        list_for_each_entry_safe (exp, texp, &ms->exportlist, explist) {
+                list_del (&exp->explist);
+                mnt3_export_free (exp);
+        }
+        ret = mnt3_init_options (ms, options);
+        UNLOCK (&ms->mountlock);
+
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Options reconfigure failed");
+                return (-1);
+        }
+
+        return (0);
+}
diff --git a/xlators/nfs/server/src/mount3.h b/xlators/nfs/server/src/mount3.h
index c0eae3644..7fc16ed57 100644
--- a/xlators/nfs/server/src/mount3.h
+++ b/xlators/nfs/server/src/mount3.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _MOUNT3_H_
@@ -53,6 +44,12 @@ mnt1svc_init (xlator_t *nfsx);
 extern int
 mount_init_state (xlator_t *nfsx);
 
+extern int
+mount_reconfigure_state (xlator_t *nfsx, dict_t *options);
+
+void
+mount_rewrite_rmtab (struct mount3_state *ms, char *new_rmtab);
+
 /* Data structure used to store the list of mounts points currently
  * in use by NFS clients.
  */
@@ -68,6 +65,13 @@ struct mountentry {
 #define MNT3_EXPTYPE_VOLUME     1
 #define MNT3_EXPTYPE_DIR        2
 
+/* Structure to hold export-dir AUTH parameter */
+struct host_auth_spec {
+        char                    *host_addr;    /* Allowed IP or host name */
+        int                     routeprefix;   /* Routing prefix */
+        struct host_auth_spec   *next;         /* Pointer to next AUTH struct */
+};
+
 struct mnt3_export {
         struct list_head        explist;
 
@@ -75,6 +79,11 @@ struct mnt3_export {
          * is exported or the subdirectory in the volume.
          */
         char                    *expname;
+        /*
+         * IP address, hostname or subnets who are allowed to connect to expname
+         * subvolume or subdirectory
+         */
+        struct host_auth_spec*  hostspec;
         xlator_t                *vol;
         int                     exptype;
 
@@ -101,8 +110,8 @@ struct mount3_state {
         gf_lock_t               mountlock;
 
         /* Set to 0 if exporting full volumes is disabled. On by default. */
-        int                     export_volumes;
-        int                     export_dirs;
+        gf_boolean_t            export_volumes;
+        gf_boolean_t            export_dirs;
 };
 
 #define gf_mnt3_export_dirs(mst)        ((mst)->export_dirs)
diff --git a/xlators/nfs/server/src/mount3udp_svc.c b/xlators/nfs/server/src/mount3udp_svc.c
new file mode 100644
index 000000000..fb59e282c
--- /dev/null
+++ b/xlators/nfs/server/src/mount3udp_svc.c
@@ -0,0 +1,189 @@
+/*
+  Copyright (c) 2012 Gluster, Inc. <http://www.gluster.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include "xdr-nfs3.h"
+#include "logging.h"
+#include "mem-pool.h"
+#include "nfs-mem-types.h"
+#include "mount3.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <rpc/pmap_clnt.h>
+#include <string.h>
+#include <memory.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+
+extern struct nfs3_fh* nfs3_rootfh (char *dp);
+extern mountres3 mnt3svc_set_mountres3 (mountstat3 stat, struct nfs3_fh *fh,
+                                        int *authflavor, u_int aflen);
+extern int
+mount3udp_add_mountlist (char *host, dirpath *expname);
+
+extern int
+mount3udp_delete_mountlist (char *host, dirpath *expname);
+
+
+/* only this thread will use this, no locking needed */
+char mnthost[INET_ADDRSTRLEN+1];
+
+mountres3 *
+mountudpproc3_mnt_3_svc(dirpath **dpp, struct svc_req *req)
+{
+        struct mountres3        *res = NULL;
+        int                     *autharr = NULL;
+        struct nfs3_fh          *fh = NULL;
+        char                    *tmp = NULL;
+
+        tmp = (char *)*dpp;
+        while (*tmp == '/')
+                tmp++;
+        fh = nfs3_rootfh (tmp);
+        if (fh == NULL) {
+                gf_log (GF_MNT, GF_LOG_DEBUG, "unable to get fh for %s", tmp);
+                goto err;
+        }
+
+        res = GF_CALLOC (1, sizeof(*res), gf_nfs_mt_mountres3);
+        if (res == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "unable to allocate memory");
+                goto err;
+        }
+        autharr = GF_CALLOC (1, sizeof(*autharr), gf_nfs_mt_int);
+        if (autharr == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "unable to allocate memory");
+                goto err;
+        }
+        autharr[0] = AUTH_UNIX;
+        *res = mnt3svc_set_mountres3 (MNT3_OK, fh, autharr, 1);
+        mount3udp_add_mountlist (mnthost, *dpp);
+        return res;
+
+ err:
+        GF_FREE (fh);
+        GF_FREE (res);
+        GF_FREE (autharr);
+        return NULL;
+}
+
+mountstat3 *
+mountudpproc3_umnt_3_svc(dirpath **dp, struct svc_req *req)
+{
+        mountstat3 *stat = NULL;
+
+        stat = GF_CALLOC (1, sizeof(mountstat3), gf_nfs_mt_mountstat3);
+        if (stat == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "unable to allocate memory");
+                return NULL;
+        }
+        *stat = MNT3_OK;
+        mount3udp_delete_mountlist (mnthost, *dp);
+        return stat;
+}
+
+static void
+mountudp_program_3(struct svc_req *rqstp, register SVCXPRT *transp)
+{
+        union {
+                dirpath mountudpproc3_mnt_3_arg;
+        } argument;
+        char                    *result = NULL;
+        xdrproc_t               _xdr_argument = NULL, _xdr_result = NULL;
+        char *(*local)(char *, struct svc_req *) = NULL;
+        mountres3               *res = NULL;
+        struct sockaddr_in      *sin = NULL;
+
+        sin = svc_getcaller (transp);
+        inet_ntop (AF_INET, &sin->sin_addr, mnthost, INET_ADDRSTRLEN+1);
+
+        switch (rqstp->rq_proc) {
+        case NULLPROC:
+                (void) svc_sendreply (transp, (xdrproc_t) xdr_void,
+                                      (char *)NULL);
+                return;
+
+        case MOUNT3_MNT:
+                _xdr_argument = (xdrproc_t) xdr_dirpath;
+                _xdr_result = (xdrproc_t) xdr_mountres3;
+                local = (char *(*)(char *,
+                                   struct svc_req *)) mountudpproc3_mnt_3_svc;
+                break;
+
+        case MOUNT3_UMNT:
+                _xdr_argument = (xdrproc_t) xdr_dirpath;
+                _xdr_result = (xdrproc_t) xdr_mountstat3;
+                local = (char *(*)(char *,
+                                   struct svc_req *)) mountudpproc3_umnt_3_svc;
+                break;
+
+        default:
+                svcerr_noproc (transp);
+                return;
+        }
+        memset ((char *)&argument, 0, sizeof (argument));
+        if (!svc_getargs (transp, (xdrproc_t) _xdr_argument,
+                          (caddr_t) &argument)) {
+                svcerr_decode (transp);
+                return;
+        }
+        result = (*local)((char *)&argument, rqstp);
+        if (result == NULL) {
+                gf_log (GF_MNT, GF_LOG_DEBUG, "PROC returned error");
+                svcerr_systemerr (transp);
+        }
+        if (result != NULL && !svc_sendreply(transp, (xdrproc_t) _xdr_result,
+                                             result)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "svc_sendreply returned error");
+                svcerr_systemerr (transp);
+        }
+        if (!svc_freeargs (transp, (xdrproc_t) _xdr_argument,
+                           (caddr_t) &argument)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "unable to free arguments");
+        }
+        if (result == NULL)
+                return;
+        /* free the result */
+        switch (rqstp->rq_proc) {
+        case MOUNT3_MNT:
+                res = (mountres3 *) result;
+                GF_FREE (res->mountres3_u.mountinfo.fhandle.fhandle3_val);
+                GF_FREE (res->mountres3_u.mountinfo.auth_flavors.auth_flavors_val);
+                GF_FREE (res);
+                break;
+
+        case MOUNT3_UMNT:
+                GF_FREE (result);
+                break;
+        }
+        return;
+}
+
+void *
+mount3udp_thread (void *argv)
+{
+        register SVCXPRT *transp = NULL;
+
+        transp = svcudp_create(RPC_ANYSOCK);
+        if (transp == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "svcudp_create error");
+                return NULL;
+        }
+        if (!svc_register(transp, MOUNT_PROGRAM, MOUNT_V3,
+                          mountudp_program_3, IPPROTO_UDP)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "svc_register error");
+                return NULL;
+        }
+
+        svc_run ();
+        gf_log (GF_MNT, GF_LOG_ERROR, "svc_run returned");
+        return NULL;
+}
diff --git a/xlators/nfs/server/src/nfs-common.c b/xlators/nfs/server/src/nfs-common.c
index b5c56d03d..f74396ee8 100644
--- a/xlators/nfs/server/src/nfs-common.c
+++ b/xlators/nfs/server/src/nfs-common.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -88,13 +79,13 @@ nfs_mntpath_to_xlator (xlator_list_t *cl, char *path)
 {
         char            volname[MNTPATHLEN];
         char            *volptr = NULL;
-        int             pathlen = 0;
+        size_t          pathlen;
         xlator_t        *targetxl = NULL;
 
         if ((!cl) || (!path))
                 return NULL;
 
-        strcpy (volname, path);
+        strncpy (volname, path, MNTPATHLEN);
         pathlen = strlen (volname);
         gf_log (GF_NFS, GF_LOG_TRACE, "Subvolume search: %s", path);
         if (volname[0] == '/')
@@ -102,7 +93,7 @@ nfs_mntpath_to_xlator (xlator_list_t *cl, char *path)
         else
                 volptr = &volname[0];
 
-        if (volname[pathlen - 1] == '/')
+        if (pathlen && volname[pathlen - 1] == '/')
                 volname[pathlen - 1] = '\0';
 
         while (cl) {
@@ -203,11 +194,26 @@ nfs_inode_loc_fill (inode_t *inode, loc_t *loc, int how)
         if ((!inode) || (!loc))
                 return ret;
 
-        ret = inode_path (inode, NULL, &resolvedpath);
-        if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "path resolution failed %s",
-                                resolvedpath);
-                goto err;
+        /* If gfid is not null, then the inode is already linked to
+         * the inode table, and not a newly created one. For newly
+         * created inode, inode_path returns null gfid as the path.
+         */
+        if (!uuid_is_null (inode->gfid)) {
+                ret = inode_path (inode, NULL, &resolvedpath);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "path resolution failed "
+                                "%s", resolvedpath);
+                        goto err;
+                }
+        }
+
+        if (resolvedpath == NULL) {
+                char tmp_path[GFID_STR_PFX_LEN + 1] = {0,};
+                snprintf (tmp_path, sizeof (tmp_path), "<gfid:%s>",
+                          uuid_utoa (loc->gfid));
+                resolvedpath = gf_strdup (tmp_path);
+        } else {
+                parent = inode_parent (inode, loc->pargfid, NULL);
         }
 
         ret = nfs_loc_fill (loc, inode, parent, resolvedpath);
@@ -222,8 +228,7 @@ err:
         if (parent)
                 inode_unref (parent);
 
-        if (resolvedpath)
-                GF_FREE (resolvedpath);
+        GF_FREE (resolvedpath);
 
         return ret;
 }
@@ -384,8 +389,7 @@ err:
         if (entryinode)
                 inode_unref (entryinode);
 
-        if (resolvedpath)
-                GF_FREE (resolvedpath);
+        GF_FREE (resolvedpath);
 
         return ret;
 }
@@ -404,6 +408,9 @@ nfs_hash_gfid (uuid_t gfid)
         uint32_t                b1 = 0;
         uint32_t                b2 = 0;
 
+        if (__is_root_gfid (gfid))
+                return 0x1;
+
         memcpy (&msb64, &gfid[8], 8);
         memcpy (&lsb64, &gfid[0], 8);
 
@@ -421,3 +428,38 @@ nfs_hash_gfid (uuid_t gfid)
 }
 
 
+void
+nfs_fix_generation (xlator_t *this, inode_t *inode)
+{
+        uint64_t                 raw_ctx        = 0;
+        struct nfs_inode_ctx    *ictx           = NULL;
+        struct nfs_state        *priv           = NULL;
+        int                      ret            = -1;
+
+        if (!inode) {
+                return;
+        }
+        priv = this->private;
+
+        if (inode_ctx_get(inode,this,&raw_ctx) == 0) {
+                ictx = (struct nfs_inode_ctx *)raw_ctx;
+                ictx->generation = priv->generation;
+        }
+        else {
+                ictx = GF_CALLOC (1, sizeof (struct nfs_inode_ctx),
+                                  gf_nfs_mt_inode_ctx);
+                if (!ictx) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not allocate nfs inode ctx");
+                        return;
+                }
+                INIT_LIST_HEAD(&ictx->shares);
+                ictx->generation = priv->generation;
+                ret = inode_ctx_put (inode, this, (uint64_t)ictx);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not store nfs inode ctx");
+                        return;
+                }
+        }
+}
diff --git a/xlators/nfs/server/src/nfs-common.h b/xlators/nfs/server/src/nfs-common.h
index 88fc14961..2e97f1563 100644
--- a/xlators/nfs/server/src/nfs-common.h
+++ b/xlators/nfs/server/src/nfs-common.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_COMMON_H_
@@ -84,4 +75,7 @@ nfs_hash_gfid (uuid_t gfid);
 
 extern int
 nfs_gfid_loc_fill (inode_table_t *itable, uuid_t gfid, loc_t *loc, int how);
+
+void
+nfs_fix_generation (xlator_t *this, inode_t *inode);
 #endif
diff --git a/xlators/nfs/server/src/nfs-fops.c b/xlators/nfs/server/src/nfs-fops.c
index 1dc07f39a..14bc0f33b 100644
--- a/xlators/nfs/server/src/nfs-fops.c
+++ b/xlators/nfs/server/src/nfs-fops.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,6 +13,9 @@
 #include "config.h"
 #endif
 
+#include <grp.h>
+#include <pwd.h>
+
 #include "dict.h"
 #include "xlator.h"
 #include "iobuf.h"
@@ -32,9 +26,81 @@
 #include "inode.h"
 #include "nfs-common.h"
 #include "nfs3-helpers.h"
+#include "nfs-mem-types.h"
 #include <libgen.h>
 #include <semaphore.h>
 
+void
+nfs_fix_groups (xlator_t *this, call_stack_t *root)
+{
+        struct passwd    mypw;
+        char             mystrs[1024];
+        struct passwd    *result;
+        gid_t            mygroups[GF_MAX_AUX_GROUPS];
+        int              ngroups;
+        int              i;
+        struct nfs_state *priv = this->private;
+        const gid_list_t *agl;
+	gid_list_t gl;
+
+        if (!priv->server_aux_gids) {
+                return;
+        }
+
+	agl = gid_cache_lookup(&priv->gid_cache, root->uid, 0, 0);
+	if (agl) {
+		for (ngroups = 0; ngroups < agl->gl_count; ngroups++) 
+			root->groups[ngroups] = agl->gl_list[ngroups];
+		root->ngrps = ngroups;
+		gid_cache_release(&priv->gid_cache, agl);
+		return;
+	}
+
+	/* No cached list found. */
+        if (getpwuid_r(root->uid,&mypw,mystrs,sizeof(mystrs),&result) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "getpwuid_r(%u) failed", root->uid);
+                return;
+        }
+
+        if (!result) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "getpwuid_r(%u) found nothing", root->uid);
+                return;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "mapped %u => %s",
+                root->uid, result->pw_name);
+
+        ngroups = GF_MAX_AUX_GROUPS;
+        if (getgrouplist(result->pw_name,root->gid,mygroups,&ngroups) == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not map %s to group list", result->pw_name);
+                return;
+        }
+
+	/* Add the group data to the cache. */
+	gl.gl_list = GF_CALLOC(ngroups, sizeof(gid_t), gf_nfs_mt_aux_gids);
+	if (gl.gl_list) {
+		/* It's not fatal if the alloc failed. */
+		gl.gl_id = root->uid;
+		gl.gl_uid = 0;
+		gl.gl_gid = 0;
+		gl.gl_count = ngroups;
+		memcpy(gl.gl_list, mygroups, sizeof(gid_t) * ngroups);
+		if (gid_cache_add(&priv->gid_cache, &gl) != 1)
+			GF_FREE(gl.gl_list);
+	}
+
+	/* Copy data to the frame. */
+        for (i = 0; i < ngroups; ++i) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s is in group %u", result->pw_name, mygroups[i]);
+                root->groups[i] = mygroups[i];
+        }
+        root->ngrps = ngroups;
+}
+
 struct nfs_fop_local *
 nfs_fop_local_init (xlator_t *nfsx)
 {
@@ -118,22 +184,34 @@ nfs_create_frame (xlator_t *xl, nfs_user_t *nfu)
         frame = create_frame (xl, (call_pool_t *)xl->ctx->pool);
         if (!frame)
                 goto err;
+	if (call_stack_alloc_groups (frame->root, nfu->ngrps) != 0) {
+		STACK_DESTROY (frame->root);
+		frame = NULL;
+		goto err;
+	}
+
         frame->root->pid = NFS_PID;
         frame->root->uid = nfu->uid;
         frame->root->gid = nfu->gids[NFS_PRIMGID_IDX];
         frame->root->lk_owner = nfu->lk_owner;
-        if (nfu->ngrps == 1)
-                goto err;       /* Done, we only got primary gid */
 
-        frame->root->ngrps = nfu->ngrps - 1;
+        if (nfu->ngrps != 1) {
+                frame->root->ngrps = nfu->ngrps - 1;
 
-        gf_log (GF_NFS, GF_LOG_TRACE,"uid: %d, gid %d, gids: %d",
-                frame->root->uid, frame->root->gid, frame->root->ngrps);
-        for(y = 0, x = 1;  y < frame->root->ngrps; x++,y++) {
-                gf_log (GF_NFS, GF_LOG_TRACE, "gid: %d", nfu->gids[x]);
-                frame->root->groups[y] = nfu->gids[x];
+                gf_log (GF_NFS, GF_LOG_TRACE,"uid: %d, gid %d, gids: %d",
+                        frame->root->uid, frame->root->gid, frame->root->ngrps);
+                for(y = 0, x = 1;  y < frame->root->ngrps; x++,y++) {
+                        gf_log (GF_NFS, GF_LOG_TRACE, "gid: %d", nfu->gids[x]);
+                        frame->root->groups[y] = nfu->gids[x];
+                }
         }
 
+        /*
+         * It's tempting to do this *instead* of using nfu above, but we need
+         * to have those values in case nfs_fix_groups doesn't do anything.
+         */
+        nfs_fix_groups(xl,frame->root);
+
 err:
         return frame;
 }
@@ -239,6 +317,9 @@ nfs_gfid_dict (inode_t *inode)
         uuid_t  rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
         dyngfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
+        if (dyngfid == NULL)
+                return (NULL);
+
         uuid_generate (newgfid);
 
         if (uuid_compare (inode->gfid, rootgfid) == 0)
@@ -249,16 +330,17 @@ nfs_gfid_dict (inode_t *inode)
         dictgfid = dict_new ();
         if (!dictgfid) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Failed to create gfid dict");
-                goto out;
+                GF_FREE (dyngfid);
+                return (NULL);
         }
 
         ret = dict_set_bin (dictgfid, "gfid-req", dyngfid, sizeof (uuid_t));
         if (ret < 0) {
+                GF_FREE (dyngfid);
                 dict_unref (dictgfid);
-                dictgfid = NULL;
+                return (NULL);
         }
 
-out:
         return dictgfid;
 }
 
@@ -274,9 +356,6 @@ out:
                 }                                                       \
         } while (0)                                                     \
 
-
-
-
 /* Fops Layer Explained
  * The fops layer has three types of functions. They can all be identified by
  * their names. Here are the three patterns:
@@ -310,6 +389,10 @@ nfs_fop_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct nfs_fop_local    *local = NULL;
         fop_lookup_cbk_t        progcbk;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (local, progcbk, frame);
         nfs_fop_restore_root_ino (local, op_ret, buf, NULL, NULL, postparent);
         if (progcbk)
@@ -353,14 +436,14 @@ err:
 
 int32_t
 nfs_fop_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_access_cbk_t        progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno);
+                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -385,7 +468,7 @@ nfs_fop_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
 
         accessbits = nfs3_request_to_accessbits (accesstest);
         STACK_WIND_COOKIE (frame, nfs_fop_access_cbk, xl, xl, xl->fops->access,
-                           loc, accessbits);
+                           loc, accessbits, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -398,7 +481,8 @@ err:
 
 int32_t
 nfs_fop_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_stat_cbk_t          progcbk = NULL;
@@ -406,7 +490,7 @@ nfs_fop_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf);
+                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -430,7 +514,7 @@ nfs_fop_stat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
         nfs_fop_save_root_ino (nfl, loc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_stat_cbk, xl, xl, xl->fops->stat,
-                           loc);
+                           loc, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -444,7 +528,8 @@ err:
 
 int32_t
 nfs_fop_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_fstat_cbk_t         progcbk = NULL;
@@ -452,7 +537,7 @@ nfs_fop_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf);
+                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -476,7 +561,7 @@ nfs_fop_fstat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfs_fop_save_root_fd_ino (nfl, fd);
 
         STACK_WIND_COOKIE (frame, nfs_fop_fstat_cbk, xl, xl, xl->fops->fstat,
-                           fd);
+                           fd, NULL);
 
         ret = 0;
 err:
@@ -491,14 +576,14 @@ err:
 
 int32_t
 nfs_fop_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd)
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_opendir_cbk_t       progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd);
+                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -520,7 +605,7 @@ nfs_fop_opendir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_opendir_cbk, xl, xl,
-                           xl->fops->opendir, pathloc, dirfd);
+                           xl->fops->opendir, pathloc, dirfd, NULL);
         ret = 0;
 
 err:
@@ -534,14 +619,14 @@ err:
 
 int
 nfs_fop_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_flush_cbk_t         progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno);
+                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -563,7 +648,7 @@ nfs_fop_flush (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_flush_cbk, xl, xl, xl->fops->flush,
-                           fd);
+                           fd, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -577,14 +662,15 @@ err:
 
 int32_t
 nfs_fop_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                      dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_readdirp_cbk_t      progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, entries);
+                progcbk (frame, cookie, this, op_ret, op_errno, entries, xdata);
 
         nfs_stack_destroy (nfl, frame);
 
@@ -624,7 +710,8 @@ err:
 
 int32_t
 nfs_fop_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                    dict_t *xdata)
 {
 
         struct nfs_fop_local    *nfl = NULL;
@@ -632,7 +719,7 @@ nfs_fop_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf);
+                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -655,7 +742,7 @@ nfs_fop_statfs (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_statfs_cbk, xl, xl,
-                           xl->fops->statfs, pathloc);
+                           xl->fops->statfs, pathloc, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -671,17 +758,21 @@ int32_t
 nfs_fop_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_create_cbk_t        progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, preparent,
                                   postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, fd, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, NULL);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -707,7 +798,7 @@ nfs_fop_create (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_create_cbk, xl, xl, xl->fops->create,
-                           pathloc, flags, mode, fd, nfl->dictgfid);
+                           pathloc, flags, mode, 0, fd, nfl->dictgfid);
 
         ret = 0;
 err:
@@ -723,7 +814,7 @@ err:
 int32_t
 nfs_fop_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                     struct iatt *post)
+                     struct iatt *post, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_setattr_cbk_t       progcbk = NULL;
@@ -731,7 +822,8 @@ nfs_fop_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, pre, post, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, pre, post);
+                progcbk (frame, cookie, this, op_ret, op_errno, pre, post,
+                         xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -756,7 +848,7 @@ nfs_fop_setattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_save_root_ino (nfl, pathloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_setattr_cbk, xl, xl,
-                           xl->fops->setattr, pathloc, buf, valid);
+                           xl->fops->setattr, pathloc, buf, valid, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -772,16 +864,20 @@ int32_t
 nfs_fop_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_mkdir_cbk_t         progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL,preparent, postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -805,7 +901,7 @@ nfs_fop_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
 
         STACK_WIND_COOKIE  (frame, nfs_fop_mkdir_cbk, xl, xl, xl->fops->mkdir,
-                            pathloc, mode, nfl->dictgfid);
+                            pathloc, mode, 0, nfl->dictgfid);
         ret = 0;
 err:
         if (ret < 0) {
@@ -821,16 +917,20 @@ int32_t
 nfs_fop_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_symlink_cbk_t       progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret,buf, NULL, preparent, postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -853,7 +953,8 @@ nfs_fop_symlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
         nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
 
         STACK_WIND_COOKIE  (frame, nfs_fop_symlink_cbk, xl, xl,
-                            xl->fops->symlink, target, pathloc, nfl->dictgfid);
+                            xl->fops->symlink, target, pathloc,
+                            0, nfl->dictgfid);
         ret = 0;
 err:
         if (ret < 0) {
@@ -868,7 +969,7 @@ err:
 int32_t
 nfs_fop_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf)
+                      struct iatt *buf, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_readlink_cbk_t      progcbk = NULL;
@@ -876,7 +977,8 @@ nfs_fop_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, path, buf);
+                progcbk (frame, cookie, this, op_ret, op_errno, path, buf,
+                         xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -899,7 +1001,7 @@ nfs_fop_readlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_save_root_ino (nfl, pathloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_readlink_cbk, xl, xl,
-                           xl->fops->readlink, pathloc, size);
+                           xl->fops->readlink, pathloc, size, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -915,16 +1017,20 @@ int32_t
 nfs_fop_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_mknod_cbk_t         progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret,buf, NULL, preparent, postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -948,7 +1054,7 @@ nfs_fop_mknod (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
 
         STACK_WIND_COOKIE  (frame, nfs_fop_mknod_cbk, xl, xl, xl->fops->mknod,
-                            pathloc, mode, dev, nfl->dictgfid);
+                            pathloc, mode, dev, 0, nfl->dictgfid);
         ret = 0;
 err:
         if (ret < 0) {
@@ -962,7 +1068,7 @@ err:
 int32_t
 nfs_fop_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = frame->local;
         fop_rmdir_cbk_t         progcbk = NULL;
@@ -972,7 +1078,7 @@ nfs_fop_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                   postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent);
+                         postparent, NULL);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -996,7 +1102,7 @@ nfs_fop_rmdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_save_root_ino (nfl, pathloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_rmdir_cbk, xl, xl, xl->fops->rmdir,
-                           pathloc, 0);
+                           pathloc, 0, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1012,7 +1118,7 @@ err:
 int32_t
 nfs_fop_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = frame->local;
         fop_unlink_cbk_t         progcbk = NULL;
@@ -1022,7 +1128,7 @@ nfs_fop_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                   postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent);
+                         postparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -1045,7 +1151,7 @@ nfs_fop_unlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         nfs_fop_save_root_ino (nfl, pathloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_unlink_cbk, xl, xl,
-                           xl->fops->unlink, pathloc);
+                           xl->fops->unlink, pathloc, 0, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1062,17 +1168,21 @@ int32_t
 nfs_fop_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_link_cbk_t          progcbk = NULL;
 
+        if (op_ret == 0) {
+                nfs_fix_generation(this,inode);
+        }
+
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, preparent,
                                   postparent);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -1097,7 +1207,7 @@ nfs_fop_link (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
         nfs_fop_save_root_ino (nfl, newloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_link_cbk, xl, xl, xl->fops->link,
-                           oldloc, newloc);
+                           oldloc, newloc, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1113,7 +1223,8 @@ int32_t
 nfs_fop_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
 
         struct nfs_fop_local    *nfl = NULL;
@@ -1131,7 +1242,7 @@ nfs_fop_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, buf,
                          preoldparent, postoldparent, prenewparent,
-                         postnewparent);
+                         postnewparent, xdata);
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -1156,7 +1267,7 @@ nfs_fop_rename (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
         nfs_fop_newloc_save_root_ino (nfl, newloc);
 
         STACK_WIND_COOKIE (frame, nfs_fop_rename_cbk, xl, xl,
-                           xl->fops->rename, oldloc, newloc);
+                           xl->fops->rename, oldloc, newloc, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1170,14 +1281,14 @@ err:
 
 int32_t
 nfs_fop_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd)
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_open_cbk_t          progcbk = NULL;
 
         nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd);
+                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
         nfs_stack_destroy (nfl, frame);
 
         return 0;
@@ -1185,7 +1296,7 @@ nfs_fop_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 nfs_fop_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              int32_t flags, fd_t *fd, int32_t wbflags, fop_open_cbk_t cbk,
+              int32_t flags, fd_t *fd, fop_open_cbk_t cbk,
               void *local)
 {
         call_frame_t            *frame = NULL;
@@ -1200,7 +1311,7 @@ nfs_fop_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
         nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
 
         STACK_WIND_COOKIE (frame, nfs_fop_open_cbk, xl, xl, xl->fops->open,
-                           loc, flags, fd, wbflags);
+                           loc, flags, fd, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1215,7 +1326,7 @@ err:
 int32_t
 nfs_fop_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf)
+                    struct iatt *postbuf, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_writev_cbk_t        progcbk = NULL;
@@ -1223,7 +1334,8 @@ nfs_fop_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,postbuf);
+                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
 
         nfs_stack_destroy (nfl, frame);
 
@@ -1257,7 +1369,7 @@ nfs_fop_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         iobref_add (nfl->iobref, srciob);
 */
         STACK_WIND_COOKIE (frame, nfs_fop_writev_cbk, xl, xl,xl->fops->writev,
-                           fd, vector, count, offset, 0, srciobref);
+                           fd, vector, count, offset, fd->flags, srciobref, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1272,7 +1384,7 @@ err:
 int32_t
 nfs_fop_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_fsync_cbk_t         progcbk = NULL;
@@ -1280,7 +1392,9 @@ nfs_fop_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,postbuf);
+                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
+
         nfs_stack_destroy (nfl, frame);
         return 0;
 }
@@ -1303,7 +1417,7 @@ nfs_fop_fsync (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfs_fop_save_root_fd_ino (nfl, fd);
 
         STACK_WIND_COOKIE (frame, nfs_fop_fsync_cbk, xl, xl,
-                           xl->fops->fsync, fd, datasync);
+                           xl->fops->fsync, fd, datasync, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1318,7 +1432,8 @@ err:
 int32_t
 nfs_fop_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                   dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_readv_cbk_t         progcbk = NULL;
@@ -1327,7 +1442,7 @@ nfs_fop_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfs_fop_restore_root_ino (nfl, op_ret, stbuf, NULL, NULL, NULL);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, vector, count,
-                         stbuf, iobref);
+                         stbuf, iobref, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -1350,7 +1465,7 @@ nfs_fop_read (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfs_fop_save_root_fd_ino (nfl, fd);
 
         STACK_WIND_COOKIE (frame, nfs_fop_readv_cbk, xl, xl, xl->fops->readv,
-                           fd, size, offset, 0);
+                           fd, size, offset, 0, NULL);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1363,7 +1478,8 @@ err:
 
 int32_t
 nfs_fop_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *flock)
+                int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_lk_cbk_t         progcbk = NULL;
@@ -1376,7 +1492,7 @@ nfs_fop_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         fd_unref (nfl->fd);
 
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, flock);
+                progcbk (frame, cookie, this, op_ret, op_errno, flock, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -1402,7 +1518,96 @@ nfs_fop_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         nfl->flock = *flock;
 
         STACK_WIND_COOKIE (frame, nfs_fop_lk_cbk, xl, xl, xl->fops->lk,
-                           fd, cmd, flock);
+                           fd, cmd, flock, NULL);
+        ret = 0;
+err:
+        if (ret < 0) {
+                if (frame)
+                        nfs_stack_destroy (nfl, frame);
+        }
+
+        return ret;
+}
+
+int32_t
+nfs_fop_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict,
+                dict_t *xdata)
+{
+        struct nfs_fop_local    *nfl = NULL;
+        fop_getxattr_cbk_t         progcbk = NULL;
+
+        nfl_to_prog_data (nfl, progcbk, frame);
+
+        if (progcbk)
+                progcbk (frame, cookie, this, op_ret, op_errno, dict, xdata);
+
+        nfs_stack_destroy (nfl, frame);
+        return 0;
+}
+
+
+int
+nfs_fop_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                  char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local)
+{
+        call_frame_t            *frame = NULL;
+        int                     ret = -EFAULT;
+        struct nfs_fop_local    *nfl = NULL;
+
+        if ((!xl) || (!loc) || (!nfu))
+                return ret;
+
+        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
+        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+
+        STACK_WIND_COOKIE (frame, nfs_fop_getxattr_cbk, xl, xl, xl->fops->getxattr,
+                           loc, name, NULL);
+        ret = 0;
+err:
+        if (ret < 0) {
+                if (frame)
+                        nfs_stack_destroy (nfl, frame);
+        }
+
+        return ret;
+}
+
+
+int32_t
+nfs_fop_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        struct nfs_fop_local    *nfl = NULL;
+        fop_setxattr_cbk_t         progcbk = NULL;
+
+        nfl_to_prog_data (nfl, progcbk, frame);
+
+        if (progcbk)
+                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
+
+        nfs_stack_destroy (nfl, frame);
+        return 0;
+}
+
+
+int
+nfs_fop_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
+                  loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
+                  fop_setxattr_cbk_t cbk, void *local)
+{
+        call_frame_t            *frame = NULL;
+        int                     ret = -EFAULT;
+        struct nfs_fop_local    *nfl = NULL;
+
+        if ((!xl) || (!loc) || (!nfu))
+                return ret;
+
+        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
+        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+
+        STACK_WIND_COOKIE (frame, nfs_fop_setxattr_cbk, xl, xl, xl->fops->setxattr,
+                           loc, dict, flags, xdata);
         ret = 0;
 err:
         if (ret < 0) {
@@ -1417,7 +1622,7 @@ err:
 int32_t
 nfs_fop_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_truncate_cbk_t      progcbk = NULL;
@@ -1425,7 +1630,8 @@ nfs_fop_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         nfl_to_prog_data (nfl, progcbk, frame);
         nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,postbuf);
+                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
 
         nfs_stack_destroy (nfl, frame);
         return 0;
@@ -1448,7 +1654,7 @@ nfs_fop_truncate (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
         nfs_fop_save_root_ino (nfl, loc);
 
         STACK_WIND_COOKIE  (frame, nfs_fop_truncate_cbk, xl, xl,
-                            xl->fops->truncate, loc, offset);
+                            xl->fops->truncate, loc, offset, NULL);
 
         ret = 0;
 err:
diff --git a/xlators/nfs/server/src/nfs-fops.h b/xlators/nfs/server/src/nfs-fops.h
index f86a66b2d..44e99c66b 100644
--- a/xlators/nfs/server/src/nfs-fops.h
+++ b/xlators/nfs/server/src/nfs-fops.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_FOPS_H_
@@ -120,7 +111,7 @@ nfs_fop_local_wipe (xlator_t *xl, struct nfs_fop_local *l);
                 nflocal = nfs_fop_local_init (nf);                      \
                 if (nflocal) {                                          \
                         nflocal->proglocal = plocal;                    \
-                        nflocal->progcbk = pcbk;                        \
+                        nflocal->progcbk = *VOID(&pcbk);                        \
                         nflocal->nfsx = nf;                             \
                         if (fram)                                       \
                                 ((call_frame_t *)fram)->local = nflocal;\
@@ -189,7 +180,7 @@ nfs_fop_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
 
 extern int
 nfs_fop_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              int32_t flags, fd_t *fd, int32_t wbflags, fop_open_cbk_t cbk,
+              int32_t flags, fd_t *fd, fop_open_cbk_t cbk,
               void *local);
 
 extern int
@@ -245,4 +236,13 @@ extern int
 nfs_fop_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
             int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local);
 
+extern int
+nfs_fop_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                  char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local);
+
+extern int
+nfs_fop_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
+                  loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
+                  fop_setxattr_cbk_t cbk, void *local);
+
 #endif
diff --git a/xlators/nfs/server/src/nfs-generics.c b/xlators/nfs/server/src/nfs-generics.c
index dbb79fc9c..cb32b7f1b 100644
--- a/xlators/nfs/server/src/nfs-generics.c
+++ b/xlators/nfs/server/src/nfs-generics.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -166,6 +157,22 @@ nfs_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
 }
 
 int
+nfs_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+              char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local)
+{
+        return nfs_fop_getxattr (nfsx, xl, nfu, loc, name, xdata, cbk, local);
+}
+
+int
+nfs_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
+              fop_setxattr_cbk_t cbk, void *local)
+{
+        return nfs_fop_setxattr (nfsx, xl, nfu, loc, dict, flags, xdata, cbk,
+                                local);
+}
+
+int
 nfs_fsync (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
            int32_t datasync, fop_fsync_cbk_t cbk, void *local)
 {
@@ -192,7 +199,7 @@ nfs_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
         if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
                 return ret;
 
-        ret = nfs_inode_open (nfsx, xl, nfu, pathloc, flags, GF_OPEN_NOWB, cbk,
+        ret = nfs_inode_open (nfsx, xl, nfu, pathloc, flags, cbk,
                               local);
         return ret;
 }
diff --git a/xlators/nfs/server/src/nfs-generics.h b/xlators/nfs/server/src/nfs-generics.h
index 11f191f5f..01876d68e 100644
--- a/xlators/nfs/server/src/nfs-generics.h
+++ b/xlators/nfs/server/src/nfs-generics.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_GENERICS_H_
@@ -164,4 +155,14 @@ nfs_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
 extern int
 nfs_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
         int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local);
+
+extern int
+nfs_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+              char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local);
+
+extern int
+nfs_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
+              fop_setxattr_cbk_t cbk, void *local);
+
 #endif
diff --git a/xlators/nfs/server/src/nfs-inodes.c b/xlators/nfs/server/src/nfs-inodes.c
index 4867dd71d..63d5e8a19 100644
--- a/xlators/nfs/server/src/nfs-inodes.c
+++ b/xlators/nfs/server/src/nfs-inodes.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -36,7 +27,7 @@
         do {                                                            \
                 nflocal = fram->local;                                  \
                 fram->local = nflocal->proglocal;                       \
-                pcbk = nflocal->progcbk;                                \
+                *VOID(&pcbk) = nflocal->progcbk;                                \
                 nfs_fop_local_wipe (nflocal->nfsx, nflocal);            \
         } while (0)                                                     \
 
@@ -57,10 +48,10 @@ nfl_inodes_init (struct nfs_fop_local *nfl, inode_t *inode, inode_t *parent,
                 nfl->newparent = inode_ref (newparent);
 
         if (name)
-                strcpy (nfl->path, name);
+                strncpy (nfl->path, name, NFS_NAME_MAX);
 
         if (newname)
-                strcpy (nfl->newpath, newname);
+                strncpy (nfl->newpath, newname, NFS_NAME_MAX);
 
         return;
 }
@@ -70,7 +61,7 @@ int32_t
 nfs_inode_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode
                       , struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent)
+                      struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = frame->local;
         fop_create_cbk_t        progcbk = NULL;
@@ -90,7 +81,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, fd, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -142,7 +133,7 @@ int32_t
 nfs_inode_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = frame->local;
         fop_mkdir_cbk_t         progcbk = NULL;
@@ -157,7 +148,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -193,7 +184,7 @@ err:
 
 int32_t
 nfs_inode_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
 
         struct nfs_fop_local    *nfl = NULL;
@@ -209,14 +200,14 @@ nfs_inode_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 */
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd);
+                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
 
 int
 nfs_inode_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                int32_t flags, int32_t wbflags, fop_open_cbk_t cbk, void *local)
+                int32_t flags, fop_open_cbk_t cbk, void *local)
 {
         struct nfs_fop_local    *nfl = NULL;
         fd_t                    *newfd = NULL;
@@ -233,7 +224,7 @@ nfs_inode_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
         }
 
         nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, fd_err);
-        ret = nfs_fop_open (nfsx, xl, nfu, loc, flags, newfd, wbflags,
+        ret = nfs_fop_open (nfsx, xl, nfu, loc, flags, newfd,
                             nfs_inode_open_cbk, nfl);
 
         if (ret < 0)
@@ -255,7 +246,8 @@ int32_t
 nfs_inode_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *buf,
                       struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent)
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_rename_cbk_t        progcbk = NULL;
@@ -272,7 +264,7 @@ do_not_link:
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, buf,
                          preoldparent, postoldparent, prenewparent,
-                         postnewparent);
+                         postnewparent, xdata);
         return 0;
 }
 
@@ -305,7 +297,7 @@ int32_t
 nfs_inode_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_link_cbk_t          progcbk = NULL;
@@ -321,7 +313,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -358,7 +350,7 @@ err:
 int32_t
 nfs_inode_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                      struct iatt *postparent)
+                      struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_unlink_cbk_t        progcbk = NULL;
@@ -375,7 +367,7 @@ do_not_unlink:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent);
+                         postparent, xdata);
         return 0;
 }
 
@@ -406,7 +398,7 @@ err:
 int32_t
 nfs_inode_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_rmdir_cbk_t         progcbk = NULL;
@@ -423,7 +415,7 @@ do_not_unlink:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                        postparent);
+                         postparent, xdata);
 
         return 0;
 }
@@ -456,7 +448,7 @@ int32_t
 nfs_inode_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_mknod_cbk_t         progcbk = NULL;
@@ -473,7 +465,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -513,7 +505,7 @@ int32_t
 nfs_inode_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, inode_t *inode,
                        struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent)
+                       struct iatt *postparent, dict_t *xdata)
 {
         struct nfs_fop_local    *nfl = NULL;
         fop_symlink_cbk_t       progcbk = NULL;
@@ -529,7 +521,7 @@ do_not_link:
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
         if (progcbk)
                 progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent);
+                         preparent, postparent, xdata);
 
         if (linked_inode) {
                 inode_lookup (linked_inode);
@@ -565,21 +557,20 @@ err:
 
 int32_t
 nfs_inode_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, fd_t *fd)
+                       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
 
         struct nfs_fop_local    *nfl = NULL;
         fop_open_cbk_t          progcbk = NULL;
 
-        if ((op_ret == -1) && (fd))
-                fd_unref (fd);
-        else
+        if (op_ret != -1)
                 fd_bind (fd);
 
         inodes_nfl_to_prog_data (nfl, progcbk, frame);
 
         if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd);
+                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
+
         return 0;
 }
 
diff --git a/xlators/nfs/server/src/nfs-inodes.h b/xlators/nfs/server/src/nfs-inodes.h
index 12c8eec0d..ba7a57124 100644
--- a/xlators/nfs/server/src/nfs-inodes.h
+++ b/xlators/nfs/server/src/nfs-inodes.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_INODES_H_
@@ -48,7 +39,7 @@ nfs_inode_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
 
 extern int
 nfs_inode_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                int32_t flags, int32_t wbflags, fop_open_cbk_t cbk,
+                int32_t flags, fop_open_cbk_t cbk,
                 void *local);
 
 extern int
diff --git a/xlators/nfs/server/src/nfs-mem-types.h b/xlators/nfs/server/src/nfs-mem-types.h
index e2dae4a34..450b6f2fe 100644
--- a/xlators/nfs/server/src/nfs-mem-types.h
+++ b/xlators/nfs/server/src/nfs-mem-types.h
@@ -2,19 +2,10 @@
    Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -37,16 +28,25 @@ enum gf_nfs_mem_types_ {
         gf_nfs_mt_entry3,
         gf_nfs_mt_entryp3,
         gf_nfs_mt_nfs3_fd_entry,
+        gf_nfs_mt_nfs3_fh,
         gf_nfs_mt_nfs_initer_list,
         gf_nfs_mt_xlator_t,
         gf_nfs_mt_list_head,
         gf_nfs_mt_mnt3_resolve,
         gf_nfs_mt_mnt3_export,
+        gf_nfs_mt_int,
+        gf_nfs_mt_mountres3,
+        gf_nfs_mt_mountstat3,
         gf_nfs_mt_inode_q,
-	gf_nfs_mt_nlm4_state,
-	gf_nfs_mt_nlm4_cm,
-	gf_nfs_mt_nlm4_fde,
+        gf_nfs_mt_nlm4_state,
+        gf_nfs_mt_nlm4_cm,
+        gf_nfs_mt_nlm4_fde,
         gf_nfs_mt_nlm4_nlmclnt,
+        gf_nfs_mt_nlm4_share,
+        gf_nfs_mt_aux_gids,
+        gf_nfs_mt_inode_ctx,
+        gf_nfs_mt_auth_spec,
+        gf_nfs_mt_arr,
         gf_nfs_mt_end
 };
 #endif
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index ba68486bd..04cf030dc 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /* This is the primary translator source for NFS.
@@ -41,6 +32,207 @@
 #include "nfs-mem-types.h"
 #include "nfs3-helpers.h"
 #include "nlm4.h"
+#include "options.h"
+#include "acl3.h"
+#include "rpc-drc.h"
+
+#define STRINGIFY(val) #val
+#define TOSTRING(val) STRINGIFY(val)
+
+#define OPT_SERVER_AUX_GIDS             "nfs.server-aux-gids"
+#define OPT_SERVER_GID_CACHE_TIMEOUT    "nfs.server.aux-gid-timeout"
+
+/* TODO: DATADIR should be based on configure's $(localstatedir) */
+#define DATADIR                         "/var/lib/glusterd"
+#define NFS_DATADIR                     DATADIR "/nfs"
+
+/* Forward declaration */
+int nfs_add_initer (struct list_head *list, nfs_version_initer_t init);
+
+static int
+nfs_init_version (xlator_t *this, nfs_version_initer_t init)
+{
+        int                       ret       = -1;
+        struct nfs_initer_list    *version  = NULL;
+        struct nfs_initer_list    *tmp      = NULL;
+        rpcsvc_program_t          *prog     = NULL;
+        struct list_head          *versions = NULL;
+        struct nfs_state          *nfs      = NULL;
+        gf_boolean_t              found     = _gf_false;
+
+        if ((!this) || (!this->private) || (!init))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        ret = nfs_add_initer (&nfs->versions, init);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Failed to add protocol initializer");
+                goto err;
+        }
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (version->init == init) {
+                        prog = init(this);
+                        if (!prog) {
+                                ret = -1;
+                                goto err;
+                        }
+                        version->program = prog;
+                        found = _gf_true;
+                        break;
+                }
+        }
+
+        /* program not added */
+        if (!found) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Program: %s NOT found", prog->progname);
+                goto err;
+        }
+
+        /* Check if nfs.port is configured */
+        if (nfs->override_portnum)
+                prog->progport = nfs->override_portnum;
+
+        gf_log (GF_NFS, GF_LOG_DEBUG, "Starting program: %s", prog->progname);
+
+        ret = rpcsvc_program_register (nfs->rpcsvc, prog);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed",
+                                                      prog->progname);
+                goto err;
+        }
+
+        /* Registration with portmapper is disabled, Nothing to do */
+        if (!nfs->register_portmap)
+                goto err;
+
+        ret = rpcsvc_program_register_portmap (prog, prog->progport);
+        if (ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Program  %s registration failed",
+                                prog->progname);
+                goto err;
+        }
+        ret = 0; /* All well */
+err:
+        return ret;
+}
+
+static int
+nfs_deinit_version (struct nfs_state *nfs, nfs_version_initer_t init)
+{
+        int                       ret       = -1;
+        struct nfs_initer_list    *version  = NULL;
+        struct nfs_initer_list    *tmp      = NULL;
+        rpcsvc_program_t          *prog     = NULL;
+        struct list_head          *versions = NULL;
+
+        if ((!nfs) || (!init))
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (version->init == init) {
+                        prog = version->program;
+                        ret = rpcsvc_program_unregister (nfs->rpcsvc, prog);
+                        if (ret != 0)
+                                return (-1);
+                        list_del (&version->list);
+                        GF_FREE (version);
+                        return (0);
+                }
+        }
+
+        return (-1);
+}
+
+static int
+nfs_reconfigure_acl3 (xlator_t *this)
+{
+        struct nfs_state          *nfs     = NULL;
+
+        if ((!this) || (!this->private))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* ACL is enabled */
+        if (nfs->enable_acl)
+                return nfs_init_version (this, acl3svc_init);
+
+        /* ACL is disabled */
+        return nfs_deinit_version (nfs, acl3svc_init);
+}
+
+static int
+nfs_reconfigure_nlm4 (xlator_t *this)
+{
+        struct nfs_state          *nfs     = NULL;
+
+        if ((!this) || (!this->private))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* NLM is enabled */
+        if (nfs->enable_nlm)
+                return nfs_init_version (this, nlm4svc_init);
+
+        /* NLM is disabled */
+        return nfs_deinit_version (nfs, nlm4svc_init);
+}
+
+static int
+nfs_program_register_portmap_all (struct nfs_state *nfs)
+{
+        struct list_head                *versions = NULL;
+        struct nfs_initer_list          *version = NULL;
+        struct nfs_initer_list          *tmp = NULL;
+        rpcsvc_program_t                *prog = NULL;
+
+        if (nfs == NULL)
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (prog == NULL)
+                        continue;
+                if (nfs->override_portnum)
+                        prog->progport = nfs->override_portnum;
+                (void) rpcsvc_program_register_portmap (prog, prog->progport);
+        }
+
+        return (0);
+}
+
+static int
+nfs_program_unregister_portmap_all (struct nfs_state *nfs)
+{
+        struct list_head                *versions = NULL;
+        struct nfs_initer_list          *version = NULL;
+        struct nfs_initer_list          *tmp = NULL;
+        rpcsvc_program_t                *prog = NULL;
+
+        if (nfs == NULL)
+                return (-1);
+
+        versions = &nfs->versions;
+        list_for_each_entry_safe (version, tmp, versions, list) {
+                prog = version->program;
+                if (prog == NULL)
+                        continue;
+                (void) rpcsvc_program_unregister_portmap (prog);
+        }
+
+        return (0);
+}
 
 /* Every NFS version must call this function with the init function
  * for its particular version.
@@ -118,7 +310,7 @@ nfs_init_versions (struct nfs_state *nfs, xlator_t *this)
                         ret = -1;
                         goto err;
                 }
-//                prog->actorxl = this;
+
                 version->program = prog;
                 if (nfs->override_portnum)
                         prog->progport = nfs->override_portnum;
@@ -127,17 +319,21 @@ nfs_init_versions (struct nfs_state *nfs, xlator_t *this)
 
                 ret = rpcsvc_program_register (nfs->rpcsvc, prog);
                 if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Program init failed");
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed",
+                                                      prog->progname);
                         goto err;
                 }
-                if (rpcsvc_register_portmap_enabled(nfs->rpcsvc)) {
+                if (nfs->register_portmap) {
                         ret = rpcsvc_program_register_portmap (prog,
                                                                prog->progport);
                         if (ret == -1) {
-                                gf_log (GF_NFS, GF_LOG_ERROR, "Program registration failed");
+                                gf_log (GF_NFS, GF_LOG_ERROR,
+                                        "Program  %s registration failed",
+                                        prog->progname);
                                 goto err;
                         }
                 }
+
         }
 
         ret = 0;
@@ -154,30 +350,41 @@ nfs_add_all_initiators (struct nfs_state *nfs)
         /* Add the initializers for all versions. */
         ret = nfs_add_initer (&nfs->versions, mnt3svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "MOUNT3 protocol initializer");
                 goto ret;
         }
 
         ret = nfs_add_initer (&nfs->versions, mnt1svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "MOUNT1 protocol initializer");
                 goto ret;
         }
 
         ret = nfs_add_initer (&nfs->versions, nfs3svc_init);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "NFS3 protocol initializer");
                 goto ret;
         }
 
-        ret = nfs_add_initer (&nfs->versions, nlm4svc_init);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                        " initializer");
-                goto ret;
+        if (nfs->enable_nlm == _gf_true) {
+                ret = nfs_add_initer (&nfs->versions, nlm4svc_init);
+                if (ret == -1) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
+                                " initializer");
+                        goto ret;
+                }
+        }
+
+        if (nfs->enable_acl == _gf_true) {
+                ret = nfs_add_initer (&nfs->versions, acl3svc_init);
+                if (ret == -1) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
+                                "ACL protocol initializer");
+                        goto ret;
+                }
         }
 
         ret = 0;
@@ -508,14 +715,16 @@ nfs_init_state (xlator_t *this)
         unsigned int            fopspoolsize = 0;
         char                    *optstr = NULL;
         gf_boolean_t            boolt = _gf_false;
+        struct stat             stbuf = {0,};
 
         if (!this)
                 return NULL;
 
-        if ((!this->children) || (!this->children->xlator)) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "nfs must have at least one"
-                        " child subvolume");
-                return NULL;
+        if (!this->children) {
+                gf_log (GF_NFS, GF_LOG_INFO,
+                        "NFS is manually disabled: Exiting");
+                /* Nothing for nfs process to do, exit cleanly */
+                kill (getpid (), SIGTERM);
         }
 
         nfs = GF_CALLOC (1, sizeof (*nfs), gf_nfs_mt_nfs_state);
@@ -570,6 +779,20 @@ nfs_init_state (xlator_t *this)
                         nfs->dynamicvolumes = GF_NFS_DVM_ON;
         }
 
+        nfs->enable_nlm = _gf_true;
+        ret = dict_get_str_boolean (this->options, "nfs.nlm", _gf_true);
+        if (ret == _gf_false) {
+                gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually disabled");
+                nfs->enable_nlm = _gf_false;
+        }
+
+        nfs->enable_acl = _gf_true;
+        ret = dict_get_str_boolean (this->options, "nfs.acl", _gf_true);
+        if (ret == _gf_false) {
+                gf_log (GF_NFS, GF_LOG_INFO, "ACL is manually disabled");
+                nfs->enable_acl = _gf_false;
+        }
+
         nfs->enable_ino32 = 0;
         if (dict_get (this->options, "nfs.enable-ino32")) {
                 ret = dict_get_str (this->options, "nfs.enable-ino32",
@@ -632,6 +855,34 @@ nfs_init_state (xlator_t *this)
                 }
         }
 
+        nfs->mount_udp = 0;
+        if (dict_get(this->options, "nfs.mount-udp")) {
+                ret = dict_get_str (this->options, "nfs.mount-udp", &optstr);
+                if (ret == -1) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
+                        goto free_foppool;
+                }
+
+                ret = gf_string2boolean (optstr, &boolt);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse bool "
+                                "string");
+                        goto free_foppool;
+                }
+
+                if (boolt == _gf_true)
+                        nfs->mount_udp = 1;
+        }
+
+        nfs->rmtab = gf_strdup (NFS_DATADIR "/rmtab");
+        if (dict_get(this->options, "nfs.mount-rmtab")) {
+                ret = dict_get_str (this->options, "nfs.mount-rmtab", &nfs->rmtab);
+                if (ret == -1) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
+                        goto free_foppool;
+                }
+        }
+
         /* support both options rpc-auth.ports.insecure and
          * rpc-auth-allow-insecure for backward compatibility
          */
@@ -692,15 +943,44 @@ nfs_init_state (xlator_t *this)
                 }
         }
 
-        nfs->rpcsvc =  rpcsvc_init (this, this->ctx, this->options, 0);
+        GF_OPTION_INIT (OPT_SERVER_AUX_GIDS, nfs->server_aux_gids,
+                        bool, free_foppool);
+        GF_OPTION_INIT (OPT_SERVER_GID_CACHE_TIMEOUT, nfs->server_aux_gids_max_age,
+                        uint32, free_foppool);
+
+        if (gid_cache_init(&nfs->gid_cache, nfs->server_aux_gids_max_age) < 0) {
+                gf_log(GF_NFS, GF_LOG_ERROR, "Failed to initialize group cache.");
+                goto free_foppool;
+        }
+
+        if (stat("/sbin/rpc.statd", &stbuf) == -1) {
+                gf_log (GF_NFS, GF_LOG_WARNING, "/sbin/rpc.statd not found. "
+                        "Disabling NLM");
+                nfs->enable_nlm = _gf_false;
+        }
+
+        nfs->rpcsvc =  rpcsvc_init (this, this->ctx,
+                                    this->options, fopspoolsize);
         if (!nfs->rpcsvc) {
                 ret = -1;
                 gf_log (GF_NFS, GF_LOG_ERROR, "RPC service init failed");
                 goto free_foppool;
         }
 
+        ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc,
+                                  this->options,
+                                  RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
+        if (ret < 0) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "Failed to configure outstanding-rpc-limit");
+                goto free_foppool;
+        }
+
+        nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc);
+
         this->private = (void *)nfs;
         INIT_LIST_HEAD (&nfs->versions);
+        nfs->generation = 1965;
 
         ret = 0;
 
@@ -719,65 +999,328 @@ free_rpcsvc:
         return nfs;
 }
 
+int
+nfs_drc_init (xlator_t *this)
+{
+        int       ret     = -1;
+        rpcsvc_t *svc     = NULL;
+
+        GF_VALIDATE_OR_GOTO (GF_NFS, this, out);
+        GF_VALIDATE_OR_GOTO (GF_NFS, this->private, out);
+
+        svc = ((struct nfs_state *)(this->private))->rpcsvc;
+        if (!svc)
+                goto out;
+
+        ret = rpcsvc_drc_init (svc, this->options);
+
+ out:
+        return ret;
+}
 
 int
+nfs_reconfigure_state (xlator_t *this, dict_t *options)
+{
+        int                 ret = 0;
+        int                 keyindx = 0;
+        char                *optstr = NULL;
+        gf_boolean_t        optbool;
+        uint32_t            optuint32;
+        struct nfs_state    *nfs = NULL;
+        char                *blacklist_keys[] = {
+                                        "nfs.port",
+                                        "nfs.transport-type",
+                                        "nfs.mem-factor",
+                                        NULL};
+
+        GF_VALIDATE_OR_GOTO (GF_NFS, this, out);
+        GF_VALIDATE_OR_GOTO (GF_NFS, this->private, out);
+        GF_VALIDATE_OR_GOTO (GF_NFS, options, out);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* Black listed options can't be reconfigured, they need
+         * NFS to be restarted. There are two cases 1. SET 2. UNSET.
+         * 1. SET */
+        while (blacklist_keys[keyindx]) {
+                if (dict_get (options, blacklist_keys[keyindx])) {
+                        gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Reconfiguring %s needs NFS restart",
+                                blacklist_keys[keyindx]);
+                        goto out;
+                }
+                keyindx ++;
+        }
+
+        /* UNSET for nfs.mem-factor */
+        if ((!dict_get (options, "nfs.mem-factor")) &&
+            (nfs->memfactor != GF_NFS_DEFAULT_MEMFACTOR)) {
+                gf_log (GF_NFS, GF_LOG_INFO,
+                        "Reconfiguring nfs.mem-factor needs NFS restart");
+                goto out;
+        }
+
+        /* UNSET for nfs.port */
+        if ((!dict_get (options, "nfs.port")) &&
+            (nfs->override_portnum)) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "Reconfiguring nfs.port needs NFS restart");
+                goto out;
+        }
+
+        /* reconfig nfs.mount-rmtab */
+        optstr = NFS_DATADIR "/rmtab";
+        if (dict_get (options, "nfs.mount-rmtab")) {
+                ret = dict_get_str (options, "nfs.mount-rmtab", &optstr);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to read "
+                                "reconfigured option: nfs.mount-rmtab");
+                        goto out;
+                }
+                gf_path_strip_trailing_slashes (optstr);
+        }
+        if (strcmp (nfs->rmtab, optstr) != 0) {
+                mount_rewrite_rmtab (nfs->mstate, optstr);
+                gf_log (GF_NFS, GF_LOG_INFO,
+                                "Reconfigured nfs.mount-rmtab path: %s",
+                                nfs->rmtab);
+        }
+
+        GF_OPTION_RECONF (OPT_SERVER_AUX_GIDS, optbool,
+                                               options, bool, out);
+        if (nfs->server_aux_gids != optbool) {
+                nfs->server_aux_gids = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d",
+                               OPT_SERVER_AUX_GIDS, optbool);
+        }
+
+        GF_OPTION_RECONF (OPT_SERVER_GID_CACHE_TIMEOUT, optuint32,
+                                               options, uint32, out);
+        if (nfs->server_aux_gids_max_age != optuint32) {
+                nfs->server_aux_gids_max_age = optuint32;
+                gid_cache_reconf (&nfs->gid_cache, optuint32);
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d",
+                               OPT_SERVER_GID_CACHE_TIMEOUT, optuint32);
+        }
+
+        /* reconfig nfs.dynamic-volumes */
+        ret = dict_get_str_boolean (options, "nfs.dynamic-volumes",
+                                             GF_NFS_DVM_OFF);
+        switch (ret) {
+        case GF_NFS_DVM_ON:
+        case GF_NFS_DVM_OFF:
+                optbool = ret;
+                break;
+        default:
+                optbool = GF_NFS_DVM_OFF;
+                break;
+        }
+        if (nfs->dynamicvolumes != optbool) {
+                nfs->dynamicvolumes = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.dynamic-volumes"
+                                            " with value %d", optbool);
+        }
+
+        optbool = _gf_false;
+        if (dict_get (options, "nfs.enable-ino32")) {
+                ret = dict_get_str_boolean (options, "nfs.enable-ino32",
+                                                      _gf_false);
+                if (ret < 0) {
+                        gf_log (GF_NFS, GF_LOG_ERROR,
+                                "Failed to read reconfigured option: "
+                                "nfs.enable-ino32");
+                        goto out;
+                }
+                optbool = ret;
+        }
+        if (nfs->enable_ino32 != optbool) {
+                nfs->enable_ino32 = optbool;
+                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.enable-ino32"
+                                            " with value %d", optbool);
+        }
+
+        /* nfs.nlm is enabled by default */
+        ret = dict_get_str_boolean (options, "nfs.nlm", _gf_true);
+        if (ret < 0) {
+                optbool = _gf_true;
+        } else {
+                optbool = ret;
+        }
+        if (nfs->enable_nlm != optbool) {
+                gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually %s",
+                                (optbool ? "enabled":"disabled"));
+                nfs->enable_nlm = optbool;
+                nfs_reconfigure_nlm4 (this);
+        }
+
+        /* nfs.acl is enabled by default */
+        ret = dict_get_str_boolean (options, "nfs.acl", _gf_true);
+        if (ret < 0) {
+                optbool = _gf_true;
+        } else {
+                optbool = ret;
+        }
+        if (nfs->enable_acl != optbool) {
+                gf_log (GF_NFS, GF_LOG_INFO, "ACL is manually %s",
+                                (optbool ? "enabled":"disabled"));
+                nfs->enable_acl = optbool;
+                nfs_reconfigure_acl3 (this);
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+/*
+ * reconfigure() for NFS server xlator.
+ */
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int                      ret = 0;
+        struct nfs_state         *nfs = NULL;
+        gf_boolean_t             regpmap = _gf_true;
+
+        if ((!this) || (!this->private) || (!options))
+                return (-1);
+
+        nfs = (struct nfs_state *)this->private;
+
+        /* Reconfigure nfs options */
+        ret = nfs_reconfigure_state(this, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "nfs reconfigure state failed");
+                return (-1);
+        }
+
+        /* Reconfigure nfs3 options */
+        ret = nfs3_reconfigure_state(this, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "nfs3 reconfigure state failed");
+                return (-1);
+        }
+
+        /* Reconfigure mount options */
+        ret = mount_reconfigure_state(this, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "mount reconfigure state failed");
+                return (-1);
+        }
+
+        /* Reconfigure rpc layer */
+        ret = rpcsvc_reconfigure_options (nfs->rpcsvc, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "rpcsvc reconfigure options failed");
+                return (-1);
+        }
+
+        /* Reconfigure rpc.outstanding-rpc-limit */
+        ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc,
+                                       options,
+                                       RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
+        if (ret < 0) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "Failed to reconfigure outstanding-rpc-limit");
+                return (-1);
+        }
+
+        regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc);
+        if (nfs->register_portmap != regpmap) {
+                nfs->register_portmap = regpmap;
+                if (regpmap) {
+                        (void) nfs_program_register_portmap_all (nfs);
+                } else {
+                        (void) nfs_program_unregister_portmap_all (nfs);
+                }
+        }
+
+        /* Reconfigure drc */
+        ret = rpcsvc_drc_reconfigure (nfs->rpcsvc, options);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR,
+                        "rpcsvc DRC reconfigure failed");
+                return (-1);
+        }
+
+        return (0);
+}
+
+/* Main init() routine for NFS server xlator. It inits NFS v3 protocol
+ * and its dependent protocols e.g. ACL, MOUNT v3 (mount3), NLM and
+ * DRC.
+ *
+ * Usage: glusterfsd:
+ *            glusterfs_process_volfp() =>
+ *              glusterfs_graph_activate() =>
+ *                glusterfs_graph_init() =>
+ *                  xlator_init () => NFS init() routine
+ *
+ * If init() routine fails, the glusterfsd cleans up the NFS process
+ * by invoking cleanup_and_exit().
+ *
+ * RETURN:
+ *       0 (SUCCESS) if all protocol specific inits PASS.
+ *      -1 (FAILURE) if any of them FAILS.
+ */
+int
 init (xlator_t *this) {
 
         struct nfs_state        *nfs = NULL;
         int                     ret = -1;
 
         if (!this)
-                return -1;
+                return (-1);
 
         nfs = nfs_init_state (this);
         if (!nfs) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Failed to init nfs option");
-                return -1;
+                return (-1);
         }
 
         ret = nfs_add_all_initiators (nfs);
-        if (ret == -1) {
+        if (ret) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add initiators");
-                goto err;
+                return (-1);
         }
 
         ret = nfs_init_subvolumes (nfs, this->children);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init NFS "
-                        "exports");
-                goto err;
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init NFS exports");
+                return (-1);
         }
 
         ret = mount_init_state (this);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init Mount"
-                        "state");
-                goto err;
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init Mount state");
+                return (-1);
         }
 
         ret = nlm4_init_state (this);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init NLM"
-                        "state");
-                goto err;
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init NLM state");
+                return (-1);
         }
 
         ret = nfs_init_versions (nfs, this);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to initialize "
-                        "protocols");
-                /* Do not return an error on this. If we dont return
-                 * an error, the process keeps running and it helps
-                 * to point out where the log is by doing ps ax|grep gluster.
-                 */
-                ret = 0;
-                goto err;
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to initialize protocols");
+                return (-1);
         }
 
-        gf_log (GF_NFS, GF_LOG_INFO, "NFS service started");
-err:
+        ret = nfs_drc_init (this);
+        if (ret) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to initialize DRC");
+                return (-1);
+        }
 
-        return ret;
+        gf_log (GF_NFS, GF_LOG_INFO, "NFS service started");
+        return (0); /* SUCCESS */
 }
 
 
@@ -785,24 +1328,26 @@ int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
         xlator_t                *subvol = NULL;
+        struct nfs_state        *priv   = NULL;
 
         subvol = (xlator_t *)data;
 
         gf_log (GF_NFS, GF_LOG_TRACE, "Notification received: %d",
                 event);
-        switch (event)
-        {
-                case GF_EVENT_CHILD_UP:
-                {
-                        nfs_startup_subvolume (this, subvol);
-                        break;
-                }
 
-                case GF_EVENT_PARENT_UP:
-                {
-                        default_notify (this, GF_EVENT_PARENT_UP, data);
-                        break;
-                }
+        switch (event) {
+        case GF_EVENT_CHILD_UP:
+                nfs_startup_subvolume (this, subvol);
+                break;
+
+        case GF_EVENT_CHILD_MODIFIED:
+                priv = this->private;
+                ++(priv->generation);
+                break;
+
+        case GF_EVENT_PARENT_UP:
+                default_notify (this, GF_EVENT_PARENT_UP, data);
+                break;
         }
 
         return 0;
@@ -824,14 +1369,138 @@ fini (xlator_t *this)
 int32_t
 nfs_forget (xlator_t *this, inode_t *inode)
 {
+        uint64_t                 ctx    = 0;
+        struct nfs_inode_ctx    *ictx   = NULL;
+
+        if (inode_ctx_del (inode, this, &ctx))
+                return -1;
+
+        ictx = (struct nfs_inode_ctx *)ctx;
+        GF_FREE (ictx);
+
         return 0;
 }
 
+gf_boolean_t
+_nfs_export_is_for_vol (char *exname, char *volname)
+{
+        gf_boolean_t    ret = _gf_false;
+        char            *tmp = NULL;
+
+        tmp = exname;
+        if (tmp[0] == '/')
+                tmp++;
+
+        if (!strcmp (tmp, volname))
+                ret = _gf_true;
+
+        return ret;
+}
+
+int
+nfs_priv_to_dict (xlator_t *this, dict_t *dict)
+{
+        int                     ret = -1;
+        struct nfs_state        *priv = NULL;
+        struct mountentry       *mentry = NULL;
+        char                    *volname = NULL;
+        char                    key[1024] = {0,};
+        int                     count = 0;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, dict, out);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not get volname");
+                goto out;
+        }
+
+        list_for_each_entry (mentry, &priv->mstate->mountlist, mlist) {
+                if (!_nfs_export_is_for_vol (mentry->exname, volname))
+                        continue;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.hostname", count);
+                ret = dict_set_str (dict, key, mentry->hostname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error writing hostname to dict");
+                        goto out;
+                }
+
+                /* No connection data available yet in nfs server.
+                 * Hence, setting to 0 to prevent cli failing
+                 */
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.bytesread", count);
+                ret = dict_set_uint64 (dict, key, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error writing bytes read to dict");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.byteswrite", count);
+                ret = dict_set_uint64 (dict, key, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error writing bytes write to dict");
+                        goto out;
+                }
+
+                count++;
+        }
+
+        ret = dict_set_int32 (dict, "clientcount", count);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error writing client count to dict");
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+extern int32_t
+nlm_priv (xlator_t *this);
+
+int32_t
+nfs_priv (xlator_t *this)
+{
+        int32_t ret = -1;
+
+        /* DRC needs the global drc structure, xl is of no use to it. */
+        ret = rpcsvc_drc_priv (((struct nfs_state *)(this->private))->rpcsvc->drc);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Statedump of DRC failed");
+                goto out;
+        }
+
+        ret = nlm_priv (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Statedump of NLM failed");
+                goto out;
+        }
+ out:
+        return ret;
+}
+
+
 struct xlator_cbks cbks = {
         .forget      = nfs_forget,
 };
 
-struct xlator_fops fops = { };
+struct xlator_fops fops;
+
+struct xlator_dumpops dumpops = {
+        .priv           = nfs_priv,
+        .priv_to_dict   = nfs_priv_to_dict,
+};
 
 /* TODO: If needed, per-volume options below can be extended to be export
 + * specific also because after export-dir is introduced, a volume is not
@@ -841,29 +1510,53 @@ struct xlator_fops fops = { };
 struct volume_options options[] = {
         { .key  = {"nfs3.read-size"},
           .type = GF_OPTION_TYPE_SIZET,
-          .description = "Size in which the client should issue read requests"
-                         " to the Gluster NFSv3 server. Must be a multiple of"
-                         " 4KB."
+          .min  = GF_NFS3_RTMIN,
+          .max  = GF_NFS3_RTMAX,
+          .default_value = TOSTRING(GF_NFS3_RTPREF),
+          .description = "Size in which the client should issue read requests "
+                         "to the Gluster NFSv3 server. Must be a multiple of "
+                         "4KB (4096). Min and Max supported values are 4KB "
+                         "(4096) and 1MB (1048576) respectively. If the "
+                         "specified value is within the supported range but "
+                         "not a multiple of 4096, it is rounded up to the "
+                         "nearest multiple of 4096."
         },
         { .key  = {"nfs3.write-size"},
           .type = GF_OPTION_TYPE_SIZET,
-          .description = "Size in which the client should issue write requests"
-                         " to the Gluster NFSv3 server. Must be a multiple of"
-                         " 4KB."
+          .min  = GF_NFS3_WTMIN,
+          .max  = GF_NFS3_WTMAX,
+          .default_value = TOSTRING(GF_NFS3_WTPREF),
+          .description = "Size in which the client should issue write requests "
+                         "to the Gluster NFSv3 server. Must be a multiple of "
+                         "1KB (1024). Min and Max supported values are "
+                         "4KB (4096) and 1MB(1048576) respectively. If the "
+                         "specified value is within the supported range but "
+                         "not a multiple of 4096, it is rounded up to the "
+                         "nearest multiple of 4096."
         },
         { .key  = {"nfs3.readdir-size"},
           .type = GF_OPTION_TYPE_SIZET,
+          .min  = GF_NFS3_DTMIN,
+          .max  = GF_NFS3_DTMAX,
+          .default_value = TOSTRING(GF_NFS3_DTPREF),
           .description = "Size in which the client should issue directory "
-                         " reading requests."
+                         "reading requests to the Gluster NFSv3 server. Must "
+                         "be a multiple of 1KB (1024). Min and Max supported "
+                         "values are 4KB (4096) and 1MB (1048576) respectively."
+                         "If the specified value is within the supported range "
+                         "but not a multiple of 4096, it is rounded up to the "
+                         "nearest multiple of 4096."
         },
         { .key  = {"nfs3.*.volume-access"},
           .type = GF_OPTION_TYPE_STR,
           .value = {"read-only", "read-write"},
+          .default_value = "read-write",
           .description = "Type of access desired for this subvolume: "
                          " read-only, read-write(default)"
         },
         { .key  = {"nfs3.*.trusted-write"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "On an UNSTABLE write from client, return STABLE flag"
                          " to force client to not send a COMMIT request. In "
                          "some environments, combined with a replicated "
@@ -878,6 +1571,7 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs3.*.trusted-sync"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "All writes and COMMIT requests are treated as async."
                          " This implies that no write requests are guaranteed"
                          " to be on server disks when the write reply is "
@@ -887,25 +1581,40 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs3.*.export-dir"},
           .type = GF_OPTION_TYPE_PATH,
+          .default_value = "",
           .description = "By default, all subvolumes of nfs are exported as "
                          "individual exports. There are cases where a "
                          "subdirectory or subdirectories in the volume need to "
                          "be exported separately. This option can also be used "
                          "in conjunction with nfs3.export-volumes option to "
                          "restrict exports only to the subdirectories specified"
-                         " through this option. Must be an absolute path."
+                         " through this option. Must be an absolute path. Along"
+                         " with path allowed list of IPs/hostname can be "
+                         "associated with each subdirectory. If provided "
+                         "connection will allowed only from these IPs. By "
+                         "default connections from all IPs are allowed. "
+                         "Format: <dir>[(hostspec[|hostspec|...])][,...]. Where"
+                         " hostspec can be an IP address, hostname or an IP "
+                         "range in CIDR notation. "
+                         "e.g. /foo(192.168.1.0/24|host1|10.1.1.8),/host2."
+                         " NOTE: Care must be taken while configuring this "
+                         "option as invalid entries and/or unreachable DNS "
+                         "servers can introduce unwanted delay in all the mount"
+                         " calls."
         },
         { .key  = {"nfs3.export-dirs"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "By default, all subvolumes of nfs are exported as "
                          "individual exports. There are cases where a "
                          "subdirectory or subdirectories in the volume need to "
                          "be exported separately. Enabling this option allows "
                          "any directory on a volumes to be exported separately."
-                         " Directory exports are enabled by default."
+                         "Directory exports are enabled by default."
         },
         { .key  = {"nfs3.export-volumes"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Enable or disable exporting whole volumes, instead "
                          "if used in conjunction with nfs3.export-dir, can "
                          "allow setting up only subdirectories as exports. On "
@@ -913,37 +1622,42 @@ struct volume_options options[] = {
         },
         { .key  = {"rpc-auth.auth-unix"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_UNIX authentication type."
-                         "Must always be enabled for better interoperability."
-                         "However, can be disabled if needed. Enabled by"
+                         "Must always be enabled for better interoperability. "
+                         "However, can be disabled if needed. Enabled by "
                          "default"
         },
         { .key  = {"rpc-auth.auth-null"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_NULL authentication type."
                          "Must always be enabled. This option is here only to"
                          " avoid unrecognized option warnings"
         },
         { .key  = {"rpc-auth.auth-unix.*"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_UNIX authentication type "
                          "for a particular exported volume overriding defaults"
                          " and general setting for AUTH_UNIX scheme. Must "
-                         "always be enabled for better interoperability."
-                         "However, can be disabled if needed. Enabled by"
+                         "always be enabled for better interoperability. "
+                         "However, can be disabled if needed. Enabled by "
                          "default."
         },
         { .key  = {"rpc-auth.auth-unix.*.allow"},
           .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_UNIX authentication type "
                          "for a particular exported volume overriding defaults"
                          " and general setting for AUTH_UNIX scheme. Must "
-                         "always be enabled for better interoperability."
-                         "However, can be disabled if needed. Enabled by"
+                         "always be enabled for better interoperability. "
+                         "However, can be disabled if needed. Enabled by "
                          "default."
         },
         { .key  = {"rpc-auth.auth-null.*"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "Disable or enable the AUTH_NULL authentication type "
                          "for a particular exported volume overriding defaults"
                          " and general setting for AUTH_NULL. Must always be "
@@ -951,35 +1665,40 @@ struct volume_options options[] = {
                          "unrecognized option warnings."
         },
         { .key  = {"rpc-auth.addr.allow"},
-          .type = GF_OPTION_TYPE_STR,
+          .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .default_value = "all",
           .description = "Allow a comma separated list of addresses and/or"
                          " hostnames to connect to the server. By default, all"
                          " connections are allowed. This allows users to "
                          "define a general rule for all exported volumes."
         },
         { .key  = {"rpc-auth.addr.reject"},
-          .type = GF_OPTION_TYPE_STR,
+          .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .default_value = "none",
           .description = "Reject a comma separated list of addresses and/or"
                          " hostnames from connecting to the server. By default,"
                          " all connections are allowed. This allows users to"
                          "define a general rule for all exported volumes."
         },
         { .key  = {"rpc-auth.addr.*.allow"},
-          .type = GF_OPTION_TYPE_STR,
+          .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .default_value = "all",
           .description = "Allow a comma separated list of addresses and/or"
                          " hostnames to connect to the server. By default, all"
                          " connections are allowed. This allows users to "
                          "define a rule for a specific exported volume."
         },
         { .key  = {"rpc-auth.addr.*.reject"},
-          .type = GF_OPTION_TYPE_STR,
+          .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .default_value = "none",
           .description = "Reject a comma separated list of addresses and/or"
                          " hostnames from connecting to the server. By default,"
-                         " all connections are allowed. This allows users to"
+                         " all connections are allowed. This allows users to "
                          "define a rule for a specific exported volume."
         },
         { .key  = {"rpc-auth.ports.insecure"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "Allow client connections from unprivileged ports. By "
                          "default only privileged ports are allowed. This is a"
                          "global setting in case insecure ports are to be "
@@ -987,31 +1706,35 @@ struct volume_options options[] = {
         },
         { .key  = {"rpc-auth.ports.*.insecure"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "Allow client connections from unprivileged ports. By "
                          "default only privileged ports are allowed. Use this"
                          " option to enable or disable insecure ports for "
-                         "a specific subvolume and to override the global setting "
-                         " set by the previous option."
+                         "a specific subvolume and to override the global "
+                         "setting set by the previous option."
         },
         { .key  = {"rpc-auth.addr.namelookup"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "Users have the option of turning on name lookup for"
                   " incoming client connections using this option. Use this "
                   "option to turn on name lookups during address-based "
                   "authentication. Turning this on will enable you to"
-                  " use hostnames in rpc-auth.addr.* filters. In some "
+                  " use hostnames in nfs.rpc-auth-* filters. In some "
                   "setups, the name server can take too long to reply to DNS "
-                  "queries resulting in timeouts of mount requests. By default, "
-                  " name lookup is off"
+                  "queries resulting in timeouts of mount requests. By "
+                  "default, name lookup is off"
         },
         { .key  = {"nfs.dynamic-volumes"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
           .description = "Internal option set to tell gnfs to use a different"
                          " scheme for encoding file handles when DVM is being"
                          " used."
         },
         { .key  = {"nfs3.*.volume-id"},
           .type = GF_OPTION_TYPE_STR,
+          .default_value = "",
           .description = "When nfs.dynamic-volumes is set, gnfs expects every "
                          "subvolume to have this option set for it, so that "
                          "gnfs can use this option to identify the volume. "
@@ -1020,26 +1743,43 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs.enable-ino32"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
           .description = "For nfs clients or apps that do not support 64-bit "
                          "inode numbers, use this option to make NFS return "
-                         "32-bit inode numbers instead. Disabled by default, so "
-                         "NFS returns 64-bit inode numbers."
+                         "32-bit inode numbers instead. Disabled by default, so"
+                         " NFS returns 64-bit inode numbers."
         },
         { .key  = {"rpc.register-with-portmap"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
           .description = "For systems that need to run multiple nfs servers, "
                          "only one registration is possible with "
                          "portmap service. Use this option to turn off portmap "
                          "registration for Gluster NFS. On by default"
         },
+        { .key  = {"rpc.outstanding-rpc-limit"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT,
+          .max  = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT,
+          .default_value = TOSTRING(RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT),
+          .description = "Parameter to throttle the number of incoming RPC "
+                         "requests from a client. 0 means no limit (can "
+                         "potentially run out of memory)"
+        },
         { .key  = {"nfs.port"},
           .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .max  = 0xffff,
+          .default_value = TOSTRING(GF_NFS3_PORT),
           .description = "Use this option on systems that need Gluster NFS to "
                          "be associated with a non-default port number."
         },
         { .key  = {"nfs.mem-factor"},
           .type = GF_OPTION_TYPE_INT,
-          .description = "Use this option to make NFS faster on systems by "
+          .min  = 1,
+          .max  = 1024,
+          .default_value = TOSTRING(GF_NFS_DEFAULT_MEMFACTOR),
+          .description = "Use this option to make NFS be faster on systems by "
                          "using more memory. This option specifies a multiple "
                          "that determines the total amount of memory used. "
                          "Default value is 15. Increase to use more memory in "
@@ -1049,9 +1789,71 @@ struct volume_options options[] = {
         },
         { .key  = {"nfs.*.disable"},
           .type = GF_OPTION_TYPE_BOOL,
-          .description = "This option is used to start or stop NFS server"
-                         "for individual volume."
+          .default_value = "false",
+          .description = "This option is used to start or stop the NFS server "
+                         "for individual volumes."
+        },
+
+        { .key  = {"nfs.nlm"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option, if set to 'off', disables NLM server "
+                         "by not registering the service with the portmapper."
+                         " Set it to 'on' to re-enable it. Default value: 'on'"
+        },
+
+        { .key = {"nfs.mount-udp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "set the option to 'on' to enable mountd on UDP. "
+                         "Required for some Solaris and AIX NFS clients. "
+                         "The need for enabling this option often depends "
+                         "on the usage of NLM."
+        },
+        { .key = {"nfs.mount-rmtab"},
+          .type = GF_OPTION_TYPE_PATH,
+          .default_value = DATADIR "/rmtab",
+          .description = "Set the location of the cache file that is used to "
+                         "list all the NFS-clients that have connected "
+                         "through the MOUNT protocol. If this is on shared "
+                         "storage, all GlusterFS servers will update and "
+                         "output (with 'showmount') the same list."
+        },
+        { .key = {OPT_SERVER_AUX_GIDS},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Let the server look up which groups a user belongs "
+                         "to, overwriting the list passed from the client. "
+                         "This enables support for group lists longer than "
+                         "can be passed through the NFS protocol, but is not "
+                         "secure unless users and groups are well synchronized "
+                         "between clients and servers."
+        },
+        { .key = {OPT_SERVER_GID_CACHE_TIMEOUT},
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 3600,
+          .default_value = "5",
+          .description = "Number of seconds to cache auxiliary-GID data, when "
+                         OPT_SERVER_AUX_GIDS " is set."
+        },
+        { .key = {"nfs.acl"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option is used to control ACL support for NFS."
+        },
+        { .key  = {"nfs.drc"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
+          .description = "Enable Duplicate Request Cache in gNFS server to "
+                         "improve correctness of non-idempotent operations like "
+                         "write, delete, link, et al"
+        },
+        { .key  = {"nfs.drc-size"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "0x20000",
+          .description = "Sets the number of non-idempotent "
+                         "requests to cache in drc"
         },
         { .key  = {NULL} },
 };
-
diff --git a/xlators/nfs/server/src/nfs.h b/xlators/nfs/server/src/nfs.h
index 706cba86f..00c7f8046 100644
--- a/xlators/nfs/server/src/nfs.h
+++ b/xlators/nfs/server/src/nfs.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __NFS_H__
@@ -29,6 +20,7 @@
 #include "dict.h"
 #include "xlator.h"
 #include "lkowner.h"
+#include "gidcache.h"
 
 #define GF_NFS                  "nfs"
 
@@ -44,7 +36,7 @@
 #define GF_NFS_MAX_MEMFACTOR            30
 
 #define GF_NFS_DVM_ON                   1
-#define GF_NFS_DVM_OFF                  2
+#define GF_NFS_DVM_OFF                  0
 
 /* This corresponds to the max 16 number of group IDs that are sent through an
  * RPC request. Since NFS is the only one going to set this, we can be safe
@@ -65,7 +57,6 @@ struct nfs_initer_list {
         rpcsvc_program_t        *program;
 };
 
-
 struct nfs_state {
         rpcsvc_t                *rpcsvc;
         struct list_head        versions;
@@ -85,7 +76,21 @@ struct nfs_state {
         int                     enable_ino32;
         unsigned int            override_portnum;
         int                     allow_insecure;
+        int                     enable_nlm;
+        int                     enable_acl;
+        int                     mount_udp;
+        char                    *rmtab;
         struct rpc_clnt         *rpc_clnt;
+        gf_boolean_t            server_aux_gids;
+	uint32_t		server_aux_gids_max_age;
+	gid_cache_t		gid_cache;
+        uint32_t                generation;
+        gf_boolean_t            register_portmap;
+};
+
+struct nfs_inode_ctx {
+        struct list_head        shares;
+        uint32_t                generation;
 };
 
 #define gf_nfs_dvm_on(nfsstt)   (((struct nfs_state *)nfsstt)->dynamicvolumes == GF_NFS_DVM_ON)
@@ -124,4 +129,7 @@ nfs_request_primary_user_init (nfs_user_t *nfu, rpcsvc_request_t *req,
                                uid_t uid, gid_t gid);
 extern int
 nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl);
+
+extern void
+nfs_fix_groups (xlator_t *this, call_stack_t *root);
 #endif
diff --git a/xlators/nfs/server/src/nfs3-fh.c b/xlators/nfs/server/src/nfs3-fh.c
index 9aea88137..e199c56dc 100644
--- a/xlators/nfs/server/src/nfs3-fh.c
+++ b/xlators/nfs/server/src/nfs3-fh.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -46,6 +37,12 @@ nfs3_fh_validate (struct nfs3_fh *fh)
 	if (fh->ident[1] != GF_NFSFH_IDENT1)
 		return 0;
 
+       if (fh->ident[2] != GF_NFSFH_IDENT2)
+               return 0;
+
+       if (fh->ident[3] != GF_NFSFH_IDENT3)
+               return 0;
+
         return 1;
 }
 
@@ -58,8 +55,9 @@ nfs3_fh_init (struct nfs3_fh *fh, struct iatt *buf)
 
         fh->ident[0] = GF_NFSFH_IDENT0;
         fh->ident[1] = GF_NFSFH_IDENT1;
+        fh->ident[2] = GF_NFSFH_IDENT2;
+        fh->ident[3] = GF_NFSFH_IDENT3;
 
-        fh->hashcount = 0;
         uuid_copy (fh->gfid, buf->ia_gfid);
 }
 
@@ -112,99 +110,33 @@ nfs3_fh_is_root_fh (struct nfs3_fh *fh)
 }
 
 
-nfs3_hash_entry_t
-nfs3_fh_hash_entry (uuid_t gfid)
-{
-        nfs3_hash_entry_t       hash = 0;
-        int                     shiftsize = 48;
-        uint64_t                ino = 0;
-        uint64_t                gen = 0;
-        nfs3_hash_entry_t       inomsb = 0;
-        nfs3_hash_entry_t       inolsb = 0;
-        nfs3_hash_entry_t       inols23b = 0;
-
-        nfs3_hash_entry_t       genmsb = 0;
-        nfs3_hash_entry_t       genlsb = 0;
-        nfs3_hash_entry_t       genls23b = 0;
-
-        memcpy (&ino, &gfid[8], 8);
-        hash = ino;
-        while (shiftsize != 0) {
-                hash ^= (ino >> shiftsize);
-                shiftsize -= 16;
-        }
-/*
-        gf_log ("FILEHANDLE", GF_LOG_TRACE, "INO %"PRIu64, ino);
-        gf_log ("FILEHANDLE",GF_LOG_TRACE, "PRI HASH %d", hash);
-*/
-        inomsb = (ino >> 56);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inomsb %d", inomsb);
-
-        inolsb = ((ino << 56) >> 56);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inolsb %d", inolsb);
-
-        inolsb = (inolsb << 8);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inolsb to inomsb %d", inolsb);
-        inols23b = ((ino << 40) >> 48);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inols23b %d", inols23b);
-
-        inols23b = (inols23b << 8);
-//        gf_log ("FILEHDNALE", GF_LOG_TRACE, "inols23b  %d", inols23b);
-
-        memcpy (&gen, &gfid[0], 8);
-        genmsb = (gen >> 56);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inomsb %d", inomsb);
-
-        genlsb = ((gen << 56) >> 56);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inolsb %d", inolsb);
-
-        genlsb = (genlsb << 8);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inolsb to inomsb %d", inolsb);
-
-        genls23b = ((gen << 40) >> 48);
-//        gf_log ("FILEHANDLE", GF_LOG_TRACE, "inols23b %d", inols23b);
-
-        genls23b = (genls23b << 8);
-//        gf_log ("FILEHDNALE", GF_LOG_TRACE, "inols23b  %d", inols23b);
-
-        hash ^= inolsb ^ inomsb ^ inols23b ^ genmsb ^ genlsb ^ genls23b;
-        return hash;
-
-}
-
 void
-nfs3_fh_to_str (struct nfs3_fh *fh, char *str)
+nfs3_fh_to_str (struct nfs3_fh *fh, char *str, size_t len)
 {
-        char            gfid[512];
-        char            exportid[512];
+        char            gfid[GF_UUID_BUF_SIZE];
+        char            exportid[GF_UUID_BUF_SIZE];
 
         if ((!fh) || (!str))
                 return;
 
-        sprintf (str, "FH: hashcount %d, exportid %s, gfid %s",
-                 fh->hashcount, uuid_utoa_r (fh->exportid, exportid),
-                 uuid_utoa_r (fh->gfid, gfid));
+        snprintf (str, len, "FH: exportid %s, gfid %s",
+                  uuid_utoa_r (fh->exportid, exportid),
+                  uuid_utoa_r (fh->gfid, gfid));
 }
 
 void
 nfs3_log_fh (struct nfs3_fh *fh)
 {
-//        int     x = 0;
         char    gfidstr[512];
         char    exportidstr[512];
 
         if (!fh)
                 return;
 
-        gf_log ("nfs3-fh", GF_LOG_TRACE, "filehandle: hashcount %d, exportid "
-                "0x%s, gfid 0x%s", fh->hashcount,
+        gf_log ("nfs3-fh", GF_LOG_TRACE, "filehandle: exportid "
+                "0x%s, gfid 0x%s",
                  uuid_utoa_r (fh->exportid, exportidstr),
                  uuid_utoa_r (fh->gfid, gfidstr));
-/*
-        for (; x < fh->hashcount; ++x)
-                gf_log ("FILEHANDLE", GF_LOG_TRACE, "Hash %d: %d", x,
-                        fh->entryhash[x]);
-*/
 }
 
 int
@@ -216,53 +148,35 @@ nfs3_fh_build_parent_fh (struct nfs3_fh *child, struct iatt *newstat,
 
         nfs3_fh_init (newfh, newstat);
         uuid_copy (newfh->exportid, child->exportid);
-        if (newstat->ia_ino == 1)
-                goto done;
 
-        newfh->hashcount = child->hashcount - 1;
-        memcpy (newfh->entryhash, child->entryhash,
-                newfh->hashcount * GF_NFSFH_ENTRYHASH_SIZE);
+        return 0;
+}
 
-done:
-//        nfs3_log_fh (newfh);
+int
+nfs3_build_fh (inode_t *inode, uuid_t exportid, struct nfs3_fh *newfh)
+{
+        if (!newfh || !inode)
+                return -1;
 
+        newfh->ident[0] = GF_NFSFH_IDENT0;
+        newfh->ident[1] = GF_NFSFH_IDENT1;
+        newfh->ident[2] = GF_NFSFH_IDENT2;
+        newfh->ident[3] = GF_NFSFH_IDENT3;
+        uuid_copy (newfh->gfid, inode->gfid);
+        uuid_copy (newfh->exportid, exportid);
         return 0;
 }
 
-
 int
 nfs3_fh_build_child_fh (struct nfs3_fh *parent, struct iatt *newstat,
                         struct nfs3_fh *newfh)
 {
-        int             hashcount = 0;
-        int             entry = 0;
-
         if ((!parent) || (!newstat) || (!newfh))
                 return -1;
 
         nfs3_fh_init (newfh, newstat);
         uuid_copy (newfh->exportid, parent->exportid);
 
-        newfh->hashcount = parent->hashcount + 1;
-        /* Only copy the hashes that are available in the parent file
-         * handle. */
-        if (parent->hashcount > GF_NFSFH_MAXHASHES)
-                hashcount = GF_NFSFH_MAXHASHES;
-        else
-                hashcount = parent->hashcount;
-
-        memcpy (newfh->entryhash, parent->entryhash,
-                hashcount * GF_NFSFH_ENTRYHASH_SIZE);
-
-        /* Do not insert parent dir hash if there is no space left in the hash
-         * array of the child entry. */
-        if (newfh->hashcount <= GF_NFSFH_MAXHASHES) {
-                entry = newfh->hashcount - 1;
-                newfh->entryhash[entry] = nfs3_fh_hash_entry (parent->gfid);
-        }
-
-//        nfs3_log_fh (newfh);
-
         return 0;
 }
 
@@ -270,34 +184,5 @@ nfs3_fh_build_child_fh (struct nfs3_fh *parent, struct iatt *newstat,
 uint32_t
 nfs3_fh_compute_size (struct nfs3_fh *fh)
 {
-        uint32_t        fhlen = 0;
-
-        if (!fh)
-                return 0;
-
-        if (fh->hashcount <= GF_NFSFH_MAXHASHES)
-                fhlen = nfs3_fh_hashcounted_size (fh->hashcount);
-        else
-                fhlen = nfs3_fh_hashcounted_size (GF_NFSFH_MAXHASHES);
-
-        return fhlen;
-}
-
-
-/* There is no point searching at a directory level which is beyond that of
- * the hashcount given in the file handle.
- */
-int
-nfs3_fh_hash_index_is_beyond (struct nfs3_fh *fh, int hashidx)
-{
-        if (!fh)
-                return 1;
-
-        if (fh->hashcount >= hashidx)
-                return 0;
-        else
-                return 1;
-
-        return 1;
+        return GF_NFSFH_STATIC_SIZE;
 }
-
diff --git a/xlators/nfs/server/src/nfs3-fh.h b/xlators/nfs/server/src/nfs3-fh.h
index e7c0625ee..1049cdc96 100644
--- a/xlators/nfs/server/src/nfs3-fh.h
+++ b/xlators/nfs/server/src/nfs3-fh.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS_FH_H_
@@ -35,17 +26,10 @@
  * handles for now. This will change if and when we need v4. */
 #define GF_NFSFH_IDENT0         ':'
 #define GF_NFSFH_IDENT1         'O'
-#define GF_NFSFH_IDENT_SIZE     (sizeof(char) * 2)
-#define GF_NFSFH_STATIC_SIZE    (GF_NFSFH_IDENT_SIZE + (2*sizeof (uuid_t)) + sizeof (uint16_t))
-#define GF_NFSFH_MAX_HASH_BYTES (NFS3_FHSIZE - GF_NFSFH_STATIC_SIZE)
-
-/* Each hash element in the file handle is of 2 bytes thus giving
- * us theoretically 65536 unique entries in a directory.
- */
-typedef uint16_t                nfs3_hash_entry_t;
-#define GF_NFSFH_ENTRYHASH_SIZE (sizeof (nfs3_hash_entry_t))
-#define GF_NFSFH_MAXHASHES      ((int)(GF_NFSFH_MAX_HASH_BYTES / GF_NFSFH_ENTRYHASH_SIZE))
-#define nfs3_fh_hashcounted_size(hcount) (GF_NFSFH_STATIC_SIZE + (hcount * GF_NFSFH_ENTRYHASH_SIZE))
+#define GF_NFSFH_IDENT2         'G'
+#define GF_NFSFH_IDENT3         'L'
+#define GF_NFSFH_IDENT_SIZE     (sizeof(char) * 4)
+#define GF_NFSFH_STATIC_SIZE    (GF_NFSFH_IDENT_SIZE + (2*sizeof (uuid_t)))
 
 #define nfs3_fh_exportid_to_index(exprtid)      ((uint16_t)exprtid[15])
 /* ATTENTION: Change in size of the structure below should be reflected in the
@@ -54,9 +38,9 @@ typedef uint16_t                nfs3_hash_entry_t;
 struct nfs3_fh {
 
         /* Used to ensure that a bunch of bytes are actually a GlusterFS NFS
-         * file handle. Should contain ":O"
+         * file handle. Should contain ":OGL"
          */
-        char                    ident[2];
+        char                    ident[4];
 
         /* UUID that identifies an export. The value stored in exportid
          * depends on the usage of gluster nfs. If the DVM is enabled using
@@ -72,11 +56,11 @@ struct nfs3_fh {
 
         /* File/dir gfid. */
         uuid_t                  gfid;
-
-        /* Number of file/ino hash elements that follow the ino. */
-        uint16_t                hashcount;
-
-        nfs3_hash_entry_t       entryhash[GF_NFSFH_MAXHASHES];
+        /* This structure must be exactly NFS3_FHSIZE (64) bytes long.
+           Having the structure shorter results in buffer overflows
+           during XDR decoding.
+        */
+        unsigned char padding[NFS3_FHSIZE - GF_NFSFH_STATIC_SIZE];
 } __attribute__((__packed__));
 
 #define GF_NFS3FH_STATIC_INITIALIZER    {{0},}
@@ -84,9 +68,6 @@ struct nfs3_fh {
 extern uint32_t
 nfs3_fh_compute_size (struct nfs3_fh *fh);
 
-extern int
-nfs3_fh_hash_index_is_beyond (struct nfs3_fh *fh, int hashidx);
-
 extern uint16_t
 nfs3_fh_hash_entry (uuid_t gfid);
 
@@ -107,7 +88,7 @@ extern void
 nfs3_log_fh (struct nfs3_fh *fh);
 
 extern void
-nfs3_fh_to_str (struct nfs3_fh *fh, char *str);
+nfs3_fh_to_str (struct nfs3_fh *fh, char *str, size_t len);
 
 extern int
 nfs3_fh_build_parent_fh (struct nfs3_fh *child, struct iatt *newstat,
@@ -115,4 +96,8 @@ nfs3_fh_build_parent_fh (struct nfs3_fh *child, struct iatt *newstat,
 
 extern struct nfs3_fh
 nfs3_fh_build_uuid_root_fh (uuid_t volumeid);
+
+extern int
+nfs3_build_fh (inode_t *inode, uuid_t exportid, struct nfs3_fh *newfh);
+
 #endif
diff --git a/xlators/nfs/server/src/nfs3-helpers.c b/xlators/nfs/server/src/nfs3-helpers.c
index f8bf689fe..9059fc341 100644
--- a/xlators/nfs/server/src/nfs3-helpers.c
+++ b/xlators/nfs/server/src/nfs3-helpers.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -102,21 +93,18 @@ struct nfs3stat_strerror nfs3stat_strerror_table[] = {
 uint64_t
 nfs3_iatt_gfid_to_ino (struct iatt *buf)
 {
-        uuid_t          gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
-        uint64_t        ino = 0;
+        uint64_t ino  = 0;
 
         if (!buf)
                 return 0;
 
-        if ((buf->ia_ino != 1) && (uuid_compare (buf->ia_gfid, gfid) != 0)) {
-                if (gf_nfs_enable_ino32()) {
-                        ino = (uint32_t )nfs_hash_gfid (buf->ia_gfid);
-                        goto hashout;
-                }
+        if (gf_nfs_enable_ino32()) {
+                ino = (uint32_t )nfs_hash_gfid (buf->ia_gfid);
+                goto hashout;
+        }
 
-                memcpy (&ino, &buf->ia_gfid[8], sizeof (uint64_t));
-        } else
-                ino = 1;
+        /* from posix its guaranteed to send unique ino */
+        ino = buf->ia_ino;
 
 hashout:
         return ino;
@@ -268,6 +256,20 @@ nfs3_errno_to_nfsstat3 (int errnum)
         return stat;
 }
 
+/*
+ * Special case: If op_ret is -1, it's very unusual op_errno being
+ * 0 which means something came wrong from upper layer(s). If it
+ * happens by any means, then set NFS3 status to NFS3ERR_SERVERFAULT.
+ */
+inline nfsstat3
+nfs3_cbk_errno_status (int32_t op_ret, int32_t op_errno)
+{
+        if ((op_ret == -1) && (op_errno == 0)) {
+                return NFS3ERR_SERVERFAULT;
+        }
+
+        return nfs3_errno_to_nfsstat3 (op_errno);
+}
 
 void
 nfs3_fill_lookup3res_error (lookup3res *res, nfsstat3 stat,
@@ -287,6 +289,9 @@ nfs3_stat_to_fattr3 (struct iatt *buf)
 {
         fattr3          fa = {0, };
 
+        if (buf == NULL)
+                goto out;
+
         if (IA_ISDIR (buf->ia_type))
                 fa.type = NF3DIR;
         else if (IA_ISREG (buf->ia_type))
@@ -356,6 +361,7 @@ nfs3_stat_to_fattr3 (struct iatt *buf)
         fa.mtime.seconds = buf->ia_mtime;
         fa.mtime.nseconds = buf->ia_mtime_nsec;
 
+out:
         return fa;
 }
 
@@ -499,7 +505,7 @@ nfs3_fill_fsinfo3res (struct nfs3_state *nfs3, fsinfo3res *res,
         resok.wtpref = nfs3->writesize;
         resok.wtmult = GF_NFS3_WTMULT;
         resok.dtpref = nfs3->readdirsize;
-        resok.maxfilesize = GF_NFS3_MAXFILE;
+        resok.maxfilesize = GF_NFS3_MAXFILESIZE;
         resok.time_delta = tdelta;
         resok.properties = GF_NFS3_FS_PROP;
 
@@ -594,7 +600,8 @@ nfs3_request_to_accessbits (int32_t accbits)
         return acc_request;
 }
 void
-nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits)
+nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits,
+		      int32_t reqaccbits)
 {
         uint32_t        accres = 0;
 
@@ -605,7 +612,8 @@ nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits)
 
         accres = nfs3_accessbits (accbits);
 
-        res->access3res_u.resok.access = accres;
+	/* do not answer what was not asked */
+        res->access3res_u.resok.access = accres & reqaccbits;
 }
 
 void
@@ -1608,13 +1616,14 @@ err:
 
 void
 nfs3_stat_to_errstr (uint32_t xid, char *op, nfsstat3 stat, int pstat,
-                     char *errstr)
+                     char *errstr, size_t len)
 {
         if ((!op) || (!errstr))
                 return;
 
-        sprintf (errstr, "XID: %x, %s: NFS: %d(%s), POSIX: %d(%s)", xid, op,
-                 stat, nfsstat3_strerror (stat), pstat, strerror (pstat));
+        snprintf (errstr, len, "XID: %x, %s: NFS: %d(%s), POSIX: %d(%s)",
+                  xid, op,stat, nfsstat3_strerror (stat), pstat,
+                  strerror (pstat));
 }
 
 void
@@ -1622,10 +1631,10 @@ nfs3_log_common_call (uint32_t xid, char *op, struct nfs3_fh *fh)
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
 
-        nfs3_fh_to_str (fh, fhstr);
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s", xid, op,
                 fhstr);
 }
@@ -1637,9 +1646,9 @@ nfs3_log_fh_entry_call (uint32_t xid, char *op, struct nfs3_fh *fh,
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s, name: %s", xid,
                 op, fhstr, name);
 }
@@ -1652,10 +1661,10 @@ nfs3_log_rename_call (uint32_t xid, struct nfs3_fh *src, char *sname,
         char    sfhstr[1024];
         char    dfhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (src, sfhstr);
-        nfs3_fh_to_str (dst, dfhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (src, sfhstr, sizeof (sfhstr));
+        nfs3_fh_to_str (dst, dfhstr, sizeof (dfhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, RENAME: args: Src: %s, "
                 "name: %s, Dst: %s, name: %s", xid, sfhstr, sname, dfhstr,
                 dname);
@@ -1673,9 +1682,9 @@ nfs3_log_create_call (uint32_t xid, struct nfs3_fh *fh, char *name,
         char    unchkd[] = "UNCHECKED";
         char    guarded[] = "GUARDED";
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         if (mode == EXCLUSIVE)
                 modestr = exclmode;
         else if (mode == GUARDED)
@@ -1698,9 +1707,9 @@ nfs3_log_mknod_call (uint32_t xid, struct nfs3_fh *fh, char *name, int type)
         char    sock[] = "SOCK";
         char    fifo[] = "FIFO";
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         if (type == NF3CHR)
                 modestr = chr;
         else if (type == NF3BLK)
@@ -1721,9 +1730,9 @@ nfs3_log_symlink_call (uint32_t xid, struct nfs3_fh *fh, char *name, char *tgt)
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, SYMLINK: args: %s, name: %s,"
                 " target: %s", xid, fhstr, name, tgt);
 }
@@ -1736,10 +1745,10 @@ nfs3_log_link_call (uint32_t xid, struct nfs3_fh *fh, char *name,
         char    dfhstr[1024];
         char    tfhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, dfhstr);
-        nfs3_fh_to_str (tgt, tfhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, dfhstr, sizeof (dfhstr));
+        nfs3_fh_to_str (tgt, tfhstr, sizeof (tfhstr));
         gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, LINK: args: %s, name: %s,"
                 " target: %s", xid, dfhstr, name, tfhstr);
 }
@@ -1751,9 +1760,9 @@ nfs3_log_rw_call (uint32_t xid, char *op, struct nfs3_fh *fh, offset3 offt,
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
-        nfs3_fh_to_str (fh, fhstr);
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
         if (stablewrite == -1)
                 gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s, offset:"
                         " %"PRIu64",  count: %"PRIu32, xid, op, fhstr, offt,
@@ -3384,11 +3393,11 @@ void
 nfs3_log_common_res (uint32_t xid, int op, nfsstat3 stat, int pstat)
 {
         char    errstr[1024];
-	int	ll = nfs3_loglevel (op, stat);
+        int     ll = nfs3_loglevel (op, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s", errstr);
 }
 
@@ -3396,14 +3405,14 @@ void
 nfs3_log_readlink_res (uint32_t xid, nfsstat3 stat, int pstat, char *linkpath)
 {
         char    errstr[1024];
-	int	ll = nfs3_loglevel (NFS3_READLINK, stat);
+        int     ll = nfs3_loglevel (NFS3_READLINK, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
+        if (THIS->ctx->log.loglevel < ll)
+                return;
 
-        nfs3_stat_to_errstr (xid, "READLINK", stat, pstat, errstr);
+        nfs3_stat_to_errstr (xid, "READLINK", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, target: %s",
-		errstr, linkpath);
+                errstr, linkpath);
 
 }
 
@@ -3412,12 +3421,12 @@ nfs3_log_read_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
                    int is_eof, struct iovec *vec, int32_t veccount)
 {
         char    errstr[1024];
-	int	ll = GF_LOG_DEBUG;
+        int     ll = GF_LOG_DEBUG;
 
 	ll = nfs3_loglevel (NFS3_READ, stat);
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, "READ", stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, "READ", stat, pstat, errstr, sizeof (errstr));
         if (vec)
                 gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", is_eof:"
                         " %d, vector: count: %d, len: %zd", errstr, count,
@@ -3433,12 +3442,12 @@ nfs3_log_write_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
                     int stable, uint64_t wverf)
 {
         char    errstr[1024];
-	int	ll = nfs3_loglevel (NFS3_WRITE, stat);
+        int     ll = nfs3_loglevel (NFS3_WRITE, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
+        if (THIS->ctx->log.loglevel < ll)
+                 return;
 
-        nfs3_stat_to_errstr (xid, "WRITE", stat, pstat, errstr);
+        nfs3_stat_to_errstr (xid, "WRITE", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", %s,wverf: %"PRIu64
                 , errstr, count, (stable == UNSTABLE)?"UNSTABLE":"STABLE",
                 wverf);
@@ -3451,12 +3460,12 @@ nfs3_log_newfh_res (uint32_t xid, int op, nfsstat3 stat, int pstat,
 {
         char    errstr[1024];
         char    fhstr[1024];
-	int	ll = nfs3_loglevel (op, stat);
+        int     ll = nfs3_loglevel (op, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr);
-        nfs3_fh_to_str (newfh, fhstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr, sizeof (errstr));
+        nfs3_fh_to_str (newfh, fhstr, sizeof (fhstr));
 
         gf_log (GF_NFS3, nfs3_loglevel (op, stat), "%s, %s", errstr, fhstr);
 }
@@ -3467,11 +3476,11 @@ nfs3_log_readdir_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
                       count3 count, int is_eof)
 {
         char    errstr[1024];
-	int	ll = nfs3_loglevel (NFS3_READDIR, stat);
+        int     ll = nfs3_loglevel (NFS3_READDIR, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, "READDIR", stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, "READDIR", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", cverf: %"PRIu64
                 ", is_eof: %d", errstr, count, cverf, is_eof);
 }
@@ -3484,9 +3493,9 @@ nfs3_log_readdirp_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
         char    errstr[1024];
 	int	ll = nfs3_loglevel (NFS3_READDIRP, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, "READDIRPLUS", stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, "READDIRPLUS", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, dircount: %"PRIu32", maxcount: %"
                 PRIu32", cverf: %"PRIu64", is_eof: %d", errstr, dircount,
                 maxcount, cverf, is_eof);
@@ -3499,9 +3508,9 @@ nfs3_log_commit_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t wverf)
         char    errstr[1024];
 	int	ll = nfs3_loglevel (NFS3_COMMIT, stat);
 
-	if (gf_log_loglevel < ll)
-		return;
-        nfs3_stat_to_errstr (xid, "COMMIT", stat, pstat, errstr);
+        if (THIS->ctx->log.loglevel < ll)
+                return;
+        nfs3_stat_to_errstr (xid, "COMMIT", stat, pstat, errstr, sizeof (errstr));
         gf_log (GF_NFS3, ll, "%s, wverf: %"PRIu64, errstr, wverf);
 }
 
@@ -3512,10 +3521,10 @@ nfs3_log_readdir_call (uint32_t xid, struct nfs3_fh *fh, count3 dircount,
 {
         char    fhstr[1024];
 
-	if (gf_log_loglevel < GF_LOG_DEBUG)
-		return;
+        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+                return;
 
-        nfs3_fh_to_str (fh, fhstr);
+        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
 
         if (maxcount == 0)
                 gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, READDIR: args: %s,"
@@ -3548,55 +3557,6 @@ err:
         return ret;
 }
 
-#define GF_NFS3_FHRESOLVE_FOUND         1
-#define GF_NFS3_FHRESOLVE_NOTFOUND      2
-#define GF_NFS3_FHRESOLVE_DIRFOUND      3
-
-int
-nfs3_fh_resolve_check_entry (struct nfs3_fh *fh, gf_dirent_t *candidate,
-                             int hashidx)
-{
-        struct iatt             *ia = NULL;
-        int                     ret = GF_NFS3_FHRESOLVE_NOTFOUND;
-        nfs3_hash_entry_t       entryhash = 0;
-
-        if ((!fh) || (!candidate))
-                return ret;
-
-        if ((strcmp (candidate->d_name, ".") == 0) ||
-            (strcmp (candidate->d_name, "..") == 0))
-                goto found_entry;
-
-        ia = &candidate->d_stat;
-        if ((uuid_compare (candidate->d_stat.ia_gfid, fh->gfid)) == 0) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Found entry: gfid: %s, "
-                        "name: %s, hashcount %d",
-                         uuid_utoa (candidate->d_stat.ia_gfid),
-                         candidate->d_name, hashidx);
-                ret = GF_NFS3_FHRESOLVE_FOUND;
-                goto found_entry;
-        }
-
-        /* This condition ensures that we never have to be afraid of having
-         * a directory hash conflict with a file hash. The consequence of
-         * this condition is that we can now have unlimited files in a directory
-         * and upto 65536 sub-directories in a directory.
-         */
-        if (!IA_ISDIR (candidate->d_stat.ia_type))
-                goto found_entry;
-        entryhash = fh->entryhash[hashidx];
-        if (entryhash == nfs3_fh_hash_entry (ia->ia_gfid)) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Found hash match: %s: %d, "
-                        "hashidx: %d", candidate->d_name, entryhash, hashidx);
-                ret = GF_NFS3_FHRESOLVE_DIRFOUND;
-                goto found_entry;
-        }
-
-found_entry:
-
-        return ret;
-}
-
 
 int32_t
 nfs3_fh_resolve_entry_lookup_cbk (call_frame_t *frame, void *cookie,
@@ -3627,7 +3587,8 @@ nfs3_fh_resolve_entry_lookup_cbk (call_frame_t *frame, void *cookie,
                                    cs->resolvedloc.name, buf);
         if (linked_inode) {
                 inode_lookup (linked_inode);
-                inode_unref (linked_inode);
+                inode_unref (cs->resolvedloc.inode);
+                cs->resolvedloc.inode = linked_inode;
         }
 err:
         nfs3_call_resume (cs);
@@ -3635,119 +3596,6 @@ err:
 }
 
 
-int
-nfs3_fh_resolve_found_entry (nfs3_call_state_t *cs, gf_dirent_t *candidate)
-{
-        int             ret = 0;
-        nfs_user_t      nfu = {0, };
-        uuid_t          gfid = {0, };
-
-        if ((!cs) || (!candidate))
-                return -EFAULT;
-
-        uuid_copy (gfid, cs->resolvedloc.inode->gfid);
-        nfs_loc_wipe (&cs->resolvedloc);
-        ret = nfs_entry_loc_fill (cs->vol->itable, gfid, candidate->d_name,
-                                  &cs->resolvedloc, NFS_RESOLVE_CREATE);
-        if (ret == -ENOENT) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry not in itable, needs"
-                        " lookup");
-                nfs_user_root_create (&nfu);
-                ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                  nfs3_fh_resolve_entry_lookup_cbk,
-                                  cs);
-        } else {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry got from itable");
-                nfs3_call_resume (cs);
-        }
-
-        return ret;
-}
-
-
-int32_t
-nfs3_fh_resolve_parent_lookup_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, inode_t *inode,
-                                  struct iatt *buf, dict_t *xattr,
-                                  struct iatt *postparent)
-{
-        nfs3_call_state_t       *cs = NULL;
-        inode_t                 *linked_inode = NULL;
-
-        cs = frame->local;
-        cs->resolve_ret = op_ret;
-        cs->resolve_errno = op_errno;
-
-        if (op_ret == -1) {
-                gf_log (GF_NFS3, (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_ERROR),
-                        "Lookup failed: %s: %s",
-                        cs->resolvedloc.path, strerror (op_errno));
-                nfs3_call_resume (cs);
-                goto err;
-        } else
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry looked up: %s",
-                        cs->resolvedloc.path);
-
-        linked_inode = inode_link (inode, cs->resolvedloc.parent,
-                                   cs->resolvedloc.name, buf);
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-                inode_unref (linked_inode);
-        }
-        nfs3_fh_resolve_entry_hard (cs);
-
-err:
-        return 0;
-}
-
-
-int
-nfs3_fh_resolve_found_parent (nfs3_call_state_t *cs, gf_dirent_t *candidate)
-{
-        int             ret = 0;
-        nfs_user_t      nfu = {0, };
-        uuid_t          gfid = {0, };
-
-        if ((!cs) || (!candidate))
-                return -EFAULT;
-
-        uuid_copy (gfid, cs->resolvedloc.inode->gfid);
-        nfs_loc_wipe (&cs->resolvedloc);
-        ret = nfs_entry_loc_fill (cs->vol->itable, gfid, candidate->d_name,
-                                  &cs->resolvedloc, NFS_RESOLVE_CREATE);
-        if (ret == -ENOENT) {
-                nfs_user_root_create (&nfu);
-                ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                  nfs3_fh_resolve_parent_lookup_cbk,
-                                  cs);
-        } else
-                nfs3_fh_resolve_entry_hard (cs);
-
-        return ret;
-}
-
-
-int
-nfs3_fh_resolve_found (nfs3_call_state_t *cs, gf_dirent_t *candidate)
-{
-        int             ret = 0;
-
-        if ((!cs) || (!candidate))
-                return -EFAULT;
-
-        if (!cs->resolventry) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Candidate entry was found");
-                ret = nfs3_fh_resolve_found_entry (cs, candidate);
-        } else {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry's parent was found");
-                ret = nfs3_fh_resolve_found_parent (cs, candidate);
-        }
-
-        return ret;
-}
-
-
 int32_t
 nfs3_fh_resolve_inode_lookup_cbk (call_frame_t *frame, void *cookie,
                                   xlator_t *this, int32_t op_ret,
@@ -3794,80 +3642,6 @@ err:
 }
 
 
-/* Validate the depth of the dir such that we do not end up opening and
- * reading directories beyond those that are needed for resolving the file
- * handle.
- * Returns 1 if fh resolution can continue, 0 otherwise.
- */
-int
-nfs3_fh_resolve_validate_dirdepth (nfs3_call_state_t *cs)
-{
-        int     ret = 1;
-
-        if (!cs)
-                return 0;
-
-        /* This condition will generally never be hit because the
-         * hash-matching scheme will prevent us from going into a
-         * directory that is not part of the hash-array.
-         */
-        if (nfs3_fh_hash_index_is_beyond (&cs->resolvefh, cs->hashidx)) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Hash index is beyond: idx %d,"
-                        " fh idx: %d", cs->hashidx, cs->resolvefh.hashcount);
-                ret = 0;
-                goto out;
-        }
-
-        if (cs->hashidx >= GF_NFSFH_MAXHASHES) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Hash index beyond max hashes:"
-                        " hashidx %d, max: %d", cs->hashidx,
-                        GF_NFSFH_MAXHASHES);
-                ret = 0;
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-/*
- * Called in a recursive code path, so if another
- * directory was opened in an earlier call during fh resolution, we must unref
- * through this reference before opening another fd_t.
- */
-#define nfs3_fh_resolve_close_cwd(cst)                                  \
-        do {                                                            \
-                if ((cst)->resolve_dir_fd) {                            \
-                        gf_log (GF_NFS3, GF_LOG_TRACE, "resolve fd "    \
-                                "unrefing: 0x%lx, ref: %d",             \
-                                (long)(cst)->resolve_dir_fd,            \
-                                (cst)->resolve_dir_fd->refcount);       \
-                                fd_unref ((cst)->resolve_dir_fd);       \
-                }                                                       \
-        } while (0)                                                     \
-
-
-int
-nfs3_fh_resolve_determine_response (nfs3_call_state_t *cs) {
-
-        int     response = GF_NFS3_FHRESOLVE_NOTFOUND;
-
-        if (!cs)
-                return response;
-
-        if ((cs->hashmatch) && (cs->entrymatch))
-                response = GF_NFS3_FHRESOLVE_FOUND;
-        else if ((cs->hashmatch) && (!cs->entrymatch))
-                response = GF_NFS3_FHRESOLVE_DIRFOUND;
-        else if ((!cs->hashmatch) && (cs->entrymatch))
-                response = GF_NFS3_FHRESOLVE_FOUND;
-        else if ((!cs->hashmatch) && (!cs->entrymatch))
-                response = GF_NFS3_FHRESOLVE_NOTFOUND;
-
-        return response;
-}
-
 
 /* Needs no extra argument since it knows that the fh to be resolved is in
  * resolvefh and that it needs to start looking from the root.
diff --git a/xlators/nfs/server/src/nfs3-helpers.h b/xlators/nfs/server/src/nfs3-helpers.h
index 67935d143..eada24221 100644
--- a/xlators/nfs/server/src/nfs3-helpers.h
+++ b/xlators/nfs/server/src/nfs3-helpers.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS3_HELPER_H_
@@ -44,6 +35,9 @@ nfs3_extract_lookup_name (lookup3args *args);
 extern nfsstat3
 nfs3_errno_to_nfsstat3 (int errnum);
 
+extern nfsstat3
+nfs3_cbk_errno_status (int32_t, int32_t);
+
 extern void
 nfs3_fill_lookup3res (lookup3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
                       struct iatt *stbuf, struct iatt *postparent,
@@ -99,7 +93,8 @@ extern void
 nfs3_prep_access3args (access3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits);
+nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits,
+		      int32_t reqaccbits);
 
 extern char *
 nfs3_fhcache_getpath (struct nfs3_state *nfs3, struct nfs3_fh *fh);
@@ -339,4 +334,7 @@ nfs3_is_parentdir_entry (char *entry);
 uint32_t
 nfs3_request_to_accessbits (int32_t accbits);
 
+void
+nfs3_map_deviceid_to_statdev (struct iatt *ia, uint64_t deviceid);
+
 #endif
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index 1cb8d07e6..0fea135c7 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -32,6 +23,7 @@
 #include "nfs3.h"
 #include "mem-pool.h"
 #include "logging.h"
+#include "nfs-common.h"
 #include "nfs-fops.h"
 #include "nfs-inodes.h"
 #include "nfs-generics.h"
@@ -216,47 +208,60 @@ out:
 }
 
 
-#define nfs3_map_fh_to_volume(nfs3state, handle, rqst, volume, status, label) \
+#define nfs3_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
         do {                                                            \
-                char buf[256];                                          \
+                char exportid[256], gfid[256];                          \
+                rpc_transport_t *trans = NULL;                          \
                 volume = nfs3_fh_to_xlator ((nfs3state), handle);       \
                 if (!volume) {                                          \
-                        uuid_unparse (handle->exportid, buf);           \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to map " \
-                                "FH to vol, exportid=%s", buf);         \
+                        uuid_unparse (handle->exportid, exportid);      \
+                        uuid_unparse (handle->gfid, gfid);              \
+                        trans = rpcsvc_request_transport (req);         \
+                        GF_LOG_OCCASIONALLY (nfs3state->occ_logger,     \
+                                GF_NFS3, GF_LOG_ERROR, "Failed to map " \
+                                "FH to vol: client=%s, exportid=%s, "   \
+                                "gfid=%s", trans->peerinfo.identifier,  \
+                                exportid, gfid);                        \
+                        GF_LOG_OCCASIONALLY (nfs3state->occ_logger,     \
+                                GF_NFS3, GF_LOG_ERROR, "Stale nfs "     \
+                                "client %s must be trying to connect to"\
+                                " a deleted volume, please unmount it.",\
+                                trans->peerinfo.identifier);            \
                         status = NFS3ERR_STALE;                         \
                         goto label;                                     \
                 } else {                                                \
-                        gf_log (GF_NFS3, GF_LOG_TRACE, "FH to Volume: %s"\
-                                ,volume->name);                         \
-                        rpcsvc_request_set_private (req, volume);   \
+                        gf_log (GF_NFS3, GF_LOG_TRACE, "FH to Volume:"  \
+                                "%s", volume->name);                    \
+                        rpcsvc_request_set_private (req, volume);       \
                 }                                                       \
         } while (0);                                                    \
 
 
 #define nfs3_validate_gluster_fh(handle, status, errlabel)              \
         do {                                                            \
-                if ((handle)) {                                         \
-                        if (!nfs3_fh_validate (handle)) {               \
-                                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad Handle");\
-                                status = NFS3ERR_BADHANDLE;             \
-                                goto errlabel;                          \
-                        }                                               \
+                if (!nfs3_fh_validate (handle)) {                       \
+                        gf_log (GF_NFS3, GF_LOG_ERROR, "Bad Handle");   \
+                        status = NFS3ERR_BADHANDLE;                     \
+                        goto errlabel;                                  \
                 }                                                       \
         } while (0)                                                     \
 
 #define nfs3_check_fh_resolve_status(cst, nfstat, erlabl)               \
         do {                                                            \
                 xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                                \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
                 if ((cst)->resolve_ret < 0) {                           \
+                        trans = rpcsvc_request_transport (cst->req);    \
                         xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
                                                      &cst->resolvefh);  \
-                        uuid_unparse (cst->fh.gfid, gfid);              \
-                        sprintf (buf, "%s : %s", xlatorp ?              \
-                                 xlatorp->name : "ERR", gfid);          \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Unable to resolve FH"\
-                                ": %s", buf);                           \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,           \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid );                                \
+                        gf_log (GF_NFS3, GF_LOG_ERROR, "%s: %s",        \
+                                strerror(cst->resolve_errno), buf);     \
                         nfstat = nfs3_errno_to_nfsstat3 (cst->resolve_errno);\
                         goto erlabl;                                    \
                 }                                                       \
@@ -265,16 +270,20 @@ out:
 #define nfs3_check_new_fh_resolve_status(cst, nfstat, erlabl)           \
         do {                                                            \
                 xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                                \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
                 if (((cst)->resolve_ret < 0) &&                         \
                     ((cst)->resolve_errno != ENOENT)) {                 \
+                        trans = rpcsvc_request_transport (cst->req);    \
                         xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
-                                                     &cst->resolvefh);   \
-                        uuid_unparse (cst->fh.gfid, gfid);             \
-                        sprintf (buf, "%s : %s", xlatorp ?              \
-                                 xlatorp->name : "ERR", gfid);          \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Unable to resolve FH"\
-                                ": %s", buf);                           \
+                                                     &cst->resolvefh);  \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,     \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid);         \
+                        gf_log (GF_NFS3, GF_LOG_ERROR, "%s: %s",        \
+                                strerror(cst->resolve_errno), buf);     \
                         nfstat = nfs3_errno_to_nfsstat3 (cs->resolve_errno);\
                         goto erlabl;                                    \
                 }                                                       \
@@ -435,11 +444,9 @@ nfs3_call_state_wipe (nfs3_call_state_t *cs)
                 fd_unref (cs->fd);
         }
 
-        if (cs->resolventry)
-                GF_FREE (cs->resolventry);
+        GF_FREE (cs->resolventry);
 
-        if (cs->pathname)
-                GF_FREE (cs->pathname);
+        GF_FREE (cs->pathname);
 
         if (!list_empty (&cs->entries.list))
                 gf_dirent_free (&cs->entries);
@@ -542,23 +549,18 @@ nfs3svc_submit_reply (rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc)
 
         iobref = iobref_new ();
         if (!iobref) {
-                iobuf_unref (iob);
                 gf_log (GF_NFS3, GF_LOG_ERROR, "failed on iobref_new()");
                 goto ret;
         }
 
-        iobref_add (iobref, iob);
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
 
         /* Then, submit the message for transmission. */
         ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-
-        /* Now that we've done our job of handing the message to the RPC layer
-         * we can safely unref the iob in the hope that RPC layer must have
-         * ref'ed the iob on receiving into the txlist.
-         */
-        iobuf_unref (iob);
-        iobref_unref (iobref);
-
         if (ret == -1) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Reply submission failed");
                 goto ret;
@@ -566,6 +568,14 @@ nfs3svc_submit_reply (rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc)
 
         ret = 0;
 ret:
+        /* Now that we've done our job of handing the message to the RPC layer
+         * we can safely unref the iob in the hope that RPC layer must have
+         * ref'ed the iob on receiving into the txlist.
+         */
+        if (NULL != iob)
+                iobuf_unref (iob);
+        if (NULL != iobref)
+                iobref_unref (iobref);
         return ret;
 }
 
@@ -597,19 +607,14 @@ nfs3svc_submit_vector_reply (rpcsvc_request_t *req, void *arg,
                 new_iobref = 1;
         }
 
-        iobref_add (iobref, iob);
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
 
         /* Then, submit the message for transmission. */
         ret = rpcsvc_submit_message (req, &outmsg, 1, payload, vcount, iobref);
-
-        /* Now that we've done our job of handing the message to the RPC layer
-         * we can safely unref the iob in the hope that RPC layer must have
-         * ref'ed the iob on receiving into the txlist.
-         */
-        iobuf_unref (iob);
-        if (new_iobref)
-                iobref_unref (iobref);
-
         if (ret == -1) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Reply submission failed");
                 goto ret;
@@ -617,6 +622,14 @@ nfs3svc_submit_vector_reply (rpcsvc_request_t *req, void *arg,
 
         ret = 0;
 ret:
+        /* Now that we've done our job of handing the message to the RPC layer
+         * we can safely unref the iob in the hope that RPC layer must have
+         * ref'ed the iob on receiving into the txlist.
+         */
+        if (NULL != iob)
+                iobuf_unref (iob);
+        if (new_iobref)
+                iobref_unref (iobref);
         return ret;
 }
 
@@ -681,11 +694,20 @@ nfs3svc_getattr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         cs = frame->local;
 
+        /*
+         * Somewhat counter-intuitively, we don't need to look for sh-failed
+         * here. Failing this getattr will generate a new lookup from the
+         * client, and nfs_fop_lookup_cbk will detect any self-heal failures.
+         */
+
         if (op_ret == -1) {
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
+        }
+        else {
+                nfs_fix_generation(this,inode);
         }
 
         nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_GETATTR,
@@ -700,7 +722,8 @@ nfs3svc_getattr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 nfs3svc_getattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
 {
         nfsstat3                status = NFS3_OK;
         nfs3_call_state_t       *cs = NULL;
@@ -711,7 +734,7 @@ nfs3svc_getattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
         }
 
         nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_GETATTR,
@@ -731,6 +754,9 @@ nfs3_getattr_resume (void *carg)
         int                             ret = -EFAULT;
         nfs_user_t                      nfu = {0, };
         nfs3_call_state_t               *cs = NULL;
+        uint64_t                         raw_ctx = 0;
+        struct nfs_inode_ctx            *ictx = NULL;
+        struct nfs_state                *priv = NULL;
 
         if (!carg)
                 return ret;
@@ -757,9 +783,28 @@ nfs3_getattr_resume (void *carg)
 		goto nfs3err;
 	}
 
+        /*
+         * If brick state changed, we need to force a proper lookup cycle (as
+         * would happen in native protocol) to do self-heal checks. We detect
+         * this by comparing the generation number for the last successful
+         * creation/lookup on the inode to the current number, so inodes that
+         * haven't been validated since the state change are affected.
+         */
+        if (inode_ctx_get(cs->resolvedloc.inode,cs->nfsx,&raw_ctx) == 0) {
+                ictx = (struct nfs_inode_ctx *)raw_ctx;
+                priv = cs->nfsx->private;
+                if (ictx->generation != priv->generation) {
+                        ret = nfs_lookup (cs->nfsx, cs->vol, &nfu,
+                                          &cs->resolvedloc,
+                                          nfs3svc_getattr_lookup_cbk, cs);
+                        goto check_err;
+                }
+        }
+
         ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
                         nfs3svc_getattr_stat_cbk, cs);
 
+check_err:
         if (ret < 0) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Stat fop failed: %s: %s",
                         cs->oploc.path, strerror (-ret));
@@ -862,7 +907,7 @@ nfs3_setattr_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preop,
 int32_t
 nfs3svc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+                      struct iatt *postbuf, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         struct iatt             *prestat = NULL;
@@ -873,7 +918,7 @@ nfs3svc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -900,7 +945,7 @@ nfs3err:
 int32_t
 nfs3svc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *preop,
-                     struct iatt *postop)
+                     struct iatt *postop, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     ret = -1;
@@ -913,7 +958,7 @@ nfs3svc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -927,7 +972,8 @@ nfs3svc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
          * truncation and also only if this is not a directory.
          */
         if ((gf_attr_size_set (cs->setattr_valid)) &&
-            (!IA_ISDIR (postop->ia_type))) {
+            (!IA_ISDIR (postop->ia_type)) &&
+            (preop->ia_size != cs->stbuf.ia_size)) {
                 nfs_request_user_init (&nfu, cs->req);
                 ret = nfs_truncate (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
                                     cs->stbuf.ia_size, nfs3svc_truncate_cbk,cs);
@@ -954,7 +1000,8 @@ nfs3err:
 
 int32_t
 nfs3svc_setattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
 {
 
         int                     ret = -EFAULT;
@@ -967,7 +1014,7 @@ nfs3svc_setattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -1181,7 +1228,7 @@ nfs3svc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_WARNING),
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto xmit_res;
         }
 
@@ -1225,7 +1272,7 @@ nfs3svc_lookup_parentdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto xmit_res;
         }
 
@@ -1267,7 +1314,11 @@ nfs3_lookup_parentdir_resume (void *carg)
         nfs3_call_state_t               *cs = NULL;
         inode_t                         *parent = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, carg, nfs3err);
+        if (!carg) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
+                        " carg value NULL");
+                return EINVAL;
+        }
 
         cs = (nfs3_call_state_t *)carg;
         nfs3_check_fh_resolve_status (cs, stat, nfs3err);
@@ -1338,10 +1389,16 @@ nfs3_lookup_resume (void *carg)
         nfs3_call_state_t               *cs = NULL;
         struct nfs3_fh                  newfh = {{0},};
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, carg, nfs3err);
+        if (!carg) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
+                        " carg value NULL");
+                return EINVAL;
+        }
 
         cs = (nfs3_call_state_t *)carg;
         nfs3_check_fh_resolve_status (cs, stat, nfs3err);
+        cs->parent = cs->resolvefh;
+
 	if (cs->hardresolved) {
 		stat = NFS3_OK;
 		nfs3_fh_build_child_fh (&cs->parent, &cs->stbuf, &newfh);
@@ -1349,7 +1406,6 @@ nfs3_lookup_resume (void *carg)
 	}
 
         nfs_request_user_init (&nfu, cs->req);
-        cs->parent = cs->resolvefh;
         ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
                           nfs3svc_lookup_cbk, cs);
         if (ret < 0)
@@ -1394,12 +1450,8 @@ nfs3_lookup (rpcsvc_request_t *req, struct nfs3_fh *fh, int fhlen, char *name)
         nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
 
         cs->lookuptype = GF_NFS3_REVALIDATE;
-        if (!nfs3_is_parentdir_entry (name))
-                ret = nfs3_fh_resolve_and_resume (cs, fh, name,
-                                                  nfs3_lookup_resume);
-        else
-                ret = nfs3_fh_resolve_and_resume (cs, fh, NULL,
-                                                  nfs3_lookup_parentdir_resume);
+        ret = nfs3_fh_resolve_and_resume (cs, fh, name,
+                                          nfs3_lookup_resume);
 
         if (ret < 0) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "failed to start hard reslove");
@@ -1453,11 +1505,12 @@ rpcerr:
 
 
 int
-nfs3_access_reply (rpcsvc_request_t *req, nfsstat3 status, int32_t accbits)
+nfs3_access_reply (rpcsvc_request_t *req, nfsstat3 status, int32_t accbits,
+		   int32_t reqaccbits)
 {
         access3res      res;
 
-        nfs3_fill_access3res (&res, status, accbits);
+        nfs3_fill_access3res (&res, status, accbits, reqaccbits);
         nfs3svc_submit_reply (req, &res,
                               (nfs3_serializer)xdr_serialize_access3res);
         return 0;
@@ -1466,7 +1519,7 @@ nfs3_access_reply (rpcsvc_request_t *req, nfsstat3 status, int32_t accbits)
 
 int32_t
 nfs3svc_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         nfsstat3                status = NFS3_OK;
         nfs3_call_state_t       *cs = NULL;
@@ -1477,11 +1530,11 @@ nfs3svc_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
         }
         nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_ACCESS, status,
                              op_errno);
-        nfs3_access_reply (cs->req, status, op_errno);
+        nfs3_access_reply (cs->req, status, op_errno, cs->accessbits);
         nfs3_call_state_wipe (cs);
 
         return 0;
@@ -1495,7 +1548,11 @@ nfs3_access_resume (void *carg)
         nfs_user_t              nfu = {0, };
         nfs3_call_state_t       *cs = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, carg, nfs3err);
+        if (!carg) {
+                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
+                        " carg value NULL");
+                return EINVAL;
+        }
 
         cs = (nfs3_call_state_t *)carg;
         nfs3_check_fh_resolve_status (cs, stat, nfs3err);
@@ -1510,7 +1567,7 @@ nfs3err:
         if (ret < 0) {
                 nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_ACCESS,
                                      stat, -ret);
-                nfs3_access_reply (cs->req, stat, 0);
+                nfs3_access_reply (cs->req, stat, 0, 0);
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
@@ -1546,7 +1603,7 @@ nfs3err:
         if (ret < 0) {
                 nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_ACCESS,
                                      stat, -ret);
-                nfs3_access_reply (req, stat, 0);
+                nfs3_access_reply (req, stat, 0, 0);
                 nfs3_call_state_wipe (cs);
                 ret = 0;
         }
@@ -1603,7 +1660,7 @@ nfs3_readlink_reply (rpcsvc_request_t *req, nfsstat3 stat, char *path,
 int32_t
 nfs3svc_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf)
+                      struct iatt *buf, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -1613,7 +1670,7 @@ nfs3svc_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -1768,7 +1825,8 @@ nfs3_read_reply (rpcsvc_request_t *req, nfsstat3 stat, count3 count,
 int32_t
 nfs3svc_read_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     is_eof = 0;
@@ -1779,7 +1837,7 @@ nfs3svc_read_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto err;
         } else
                 stat = NFS3_OK;
@@ -1952,7 +2010,7 @@ nfs3_write_reply (rpcsvc_request_t *req, nfsstat3 stat, count3 count,
 int32_t
 nfs3svc_write_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf)
+                         struct iatt *postbuf, dict_t *xdata)
 {
         struct nfs3_state       *nfs3 = NULL;
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
@@ -1965,7 +2023,7 @@ nfs3svc_write_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else
                 stat = NFS3_OK;
 
@@ -1978,41 +2036,6 @@ nfs3svc_write_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 
-/*
- * If this logic determines that the write should return a reply to the client
- * after this function, the return value is -1 and the writetype is reset to
- * the type of write we want to signify to the client.
- *
- * In case the write should continue to serve the request according to the type
- * of stable write, a 0 is returned and writetype is left as it is.
- */
-int
-nfs3_write_how (int *writetype, int write_trusted, int sync_trusted)
-{
-        int     ret = -1;
-
-        if (*writetype == UNSTABLE) {
-                /* On an UNSTABLE write, only return STABLE when trusted-write
-                 * is set. TW is also set when trusted-sync is set.
-                 */
-                if (write_trusted)
-                        *writetype = FILE_SYNC;
-
-                goto err;
-        } else if ((*writetype == DATA_SYNC) || (*writetype == FILE_SYNC)) {
-
-                /* On a STABLE write, if sync-trusted is on, only then, return
-                 * without syncing.
-                 */
-                if (sync_trusted)
-                        goto err;
-        }
-
-        ret = 0;
-err:
-        return ret;
-}
-
 
 /*
  * Before going into the write reply logic, here is a matrix that shows the
@@ -2048,15 +2071,11 @@ err:
 int32_t
 nfs3svc_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs_user_t              nfu = {0, };
         nfs3_call_state_t       *cs = NULL;
         struct nfs3_state       *nfs3 = NULL;
-        int                     write_trusted = 0;
-        int                     sync_trusted = 0;
 
         cs = frame->local;
         nfs3 = rpcsvc_request_program_private (cs->req);
@@ -2064,42 +2083,21 @@ nfs3svc_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto err;
         }
 
         stat = NFS3_OK;
         cs->maxcount = op_ret;
 
-        write_trusted = nfs3_export_write_trusted (cs->nfs3state,
-                                                   cs->resolvefh.exportid);
-        sync_trusted = nfs3_export_sync_trusted (cs->nfs3state,
-                                                 cs->resolvefh.exportid);
-        ret = nfs3_write_how (&cs->writetype, write_trusted, sync_trusted);
-        if (ret == -1)
-                goto err;
-
-        nfs_request_user_init (&nfu, cs->req);
-        /* Store the current preattr so that this can be used as the pre attr
-         * when fsync returns. We dont want to use the preattr in fsync because
-         * the write fop happened before the fsync.
-         */
-        cs->stbuf = *prebuf;
-        ret = nfs_fsync (cs->nfsx, cs->vol, &nfu, cs->fd, 0,
-                         nfs3svc_write_fsync_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-
 err:
-        if (ret < 0) {
-                nfs3_log_write_res (rpcsvc_request_xid (cs->req), stat,
-                                    op_errno, cs->maxcount, cs->writetype,
-                                    nfs3->serverstart);
-                nfs3_write_reply (cs->req, stat, cs->maxcount,
-                                  cs->writetype, nfs3->serverstart, prebuf,
-                                  postbuf);
-                nfs3_call_state_wipe (cs);
-        }
+	nfs3_log_write_res (rpcsvc_request_xid (cs->req), stat,
+			    op_errno, cs->maxcount, cs->writetype,
+			    nfs3->serverstart);
+	nfs3_write_reply (cs->req, stat, cs->maxcount,
+			  cs->writetype, nfs3->serverstart, prebuf,
+			  postbuf);
+	nfs3_call_state_wipe (cs);
 
         return 0;
 }
@@ -2155,6 +2153,25 @@ nfs3_write_resume (void *carg)
         }
 
         cs->fd = fd;    /* Gets unrefd when the call state is wiped. */
+
+/*
+  enum stable_how {
+  UNSTABLE = 0,
+  DATA_SYNC = 1,
+  FILE_SYNC = 2,
+  };
+*/
+	switch (cs->writetype) {
+	case UNSTABLE:
+		break;
+	case DATA_SYNC:
+		fd->flags |= O_DSYNC;
+		break;
+	case FILE_SYNC:
+		fd->flags |= O_SYNC;
+		break;
+	}
+
         ret = __nfs3_write_resume (cs);
         if (ret < 0)
                 stat = nfs3_errno_to_nfsstat3 (-ret);
@@ -2225,17 +2242,18 @@ out:
 
 
 int
-nfs3svc_write_vecsizer (int state, ssize_t *readsize, char *addr)
+nfs3svc_write_vecsizer (int state, ssize_t *readsize, char *base_addr,
+                        char *curr_addr)
 {
-        int         ret = 0;
-        uint32_t        fhlen = 0;
-        uint32_t        fhlen_n = 0;
+        int      ret     = 0;
+        uint32_t fhlen   = 0;
+        uint32_t fhlen_n = 0;
 
         if (state == 0) {
                 ret = NFS3_VECWRITE_READFHLEN;
                 *readsize = 4;
         } else if (state == NFS3_VECWRITE_READFHLEN) {
-                fhlen_n = *(uint32_t *)(addr - 4);
+                fhlen_n = *(uint32_t *)(curr_addr - 4);
                 fhlen = ntohl (fhlen_n);
                 *readsize = xdr_length_round_up (fhlen, NFS3_FHSIZE);
                 ret = NFS3_VECWRITE_READFH;
@@ -2286,12 +2304,6 @@ rpcerr:
         return ret;
 }
 
-int
-nfs3svc_write_vec (rpcsvc_request_t *req, struct iovec *payload,
-                   int payload_count, struct iobref *iobref)
-{
-        return nfs3svc_write (req);
-}
 
 int
 nfs3_create_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *newfh,
@@ -2313,7 +2325,7 @@ nfs3_create_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *newfh,
 int32_t
 nfs3svc_create_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             int32_t op_ret, int32_t op_errno,
-                            struct iatt *preop, struct iatt *postop)
+                            struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -2323,7 +2335,7 @@ nfs3svc_create_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2343,7 +2355,7 @@ int32_t
 nfs3svc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     ret = -EFAULT;
@@ -2356,7 +2368,7 @@ nfs3svc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2446,7 +2458,8 @@ nfs3_create_common (nfs3_call_state_t *cs)
 
 int32_t
 nfs3svc_create_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         dict_t *xdata)
 {
         int                     ret = -EFAULT;
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
@@ -2460,7 +2473,7 @@ nfs3svc_create_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
                 ret = -op_errno;
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2631,13 +2644,7 @@ nfs3svc_create (rpcsvc_request_t *req)
         }
 
         cval = (uint64_t *)args.how.createhow3_u.verf;
-        if (cval)
-                cverf = *cval;
-        else {
-                gf_log(GF_NFS3, GF_LOG_ERROR,
-                       "Error getting createverf3 from args");
-                goto rpcerr;
-        }
+        cverf = *cval;
 
         ret = nfs3_create (req, &dirfh, name, args.how.mode,
                            &args.how.createhow3_u.obj_attributes, cverf);
@@ -2671,7 +2678,7 @@ nfs3_mkdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
 int32_t
 nfs3svc_mkdir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            int32_t op_ret, int32_t op_errno,
-                           struct iatt *preop, struct iatt *postop)
+                           struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -2681,7 +2688,7 @@ nfs3svc_mkdir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2701,7 +2708,7 @@ int32_t
 nfs3svc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                        stat = NFS3ERR_SERVERFAULT;
         int                             ret = -EFAULT;
@@ -2713,7 +2720,7 @@ nfs3svc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -2881,7 +2888,7 @@ int32_t
 nfs3svc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                        stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t               *cs = NULL;
@@ -2891,7 +2898,7 @@ nfs3svc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -3043,7 +3050,7 @@ nfs3_mknod_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
 int32_t
 nfs3svc_mknod_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            int32_t op_ret, int32_t op_errno,
-                           struct iatt *preop, struct iatt *postop)
+                           struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -3053,7 +3060,7 @@ nfs3svc_mknod_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -3073,7 +3080,7 @@ int32_t
 nfs3svc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                        stat = NFS3ERR_SERVERFAULT;
         int                             ret = -1;
@@ -3085,7 +3092,7 @@ nfs3svc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -3333,7 +3340,7 @@ nfs3_remove_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preparent
 int32_t
 nfs3svc_remove_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -3343,7 +3350,7 @@ nfs3svc_remove_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         }
 
         if (op_ret == 0)
@@ -3499,7 +3506,7 @@ nfs3_rmdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preparent,
 int32_t
 nfs3svc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -3509,7 +3516,7 @@ nfs3svc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else {
                 stat = NFS3_OK;
         }
@@ -3651,7 +3658,8 @@ int32_t
 nfs3svc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
         int                     ret = -EFAULT;
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
@@ -3663,7 +3671,7 @@ nfs3svc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%x: rename %s -> %s => -1 (%s)",
                         rpcsvc_request_xid (cs->req), cs->oploc.path,
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -3857,7 +3865,7 @@ int32_t
 nfs3svc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -3868,7 +3876,7 @@ nfs3svc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%x: link %s <- %s => -1 (%s)",
                         rpcsvc_request_xid (cs->req), cs->oploc.path,
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else
                 stat = NFS3_OK;
 
@@ -4066,7 +4074,8 @@ nfs3_readdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *dirfh,
 
 int32_t
 nfs3svc_readdir_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                           int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                           dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     is_eof = 0;
@@ -4077,7 +4086,7 @@ nfs3svc_readdir_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto nfs3err;
         }
 
@@ -4119,7 +4128,8 @@ nfs3err:
 
 int32_t
 nfs3svc_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     ret = -EFAULT;
@@ -4131,7 +4141,7 @@ nfs3svc_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto err;
         }
 
@@ -4228,12 +4238,30 @@ nfs3err:
 }
 
 
+int32_t
+nfs3svc_readdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, fd_t *fd,
+                             dict_t *xdata)
+{
+        /*
+         * We don't really need this, it's just an artifact of forcing the
+         * opendir to happen.
+         */
+        if (fd) {
+                fd_unref(fd);
+        }
+
+        return 0;
+}
+
+
 int
 nfs3_readdir_open_resume (void *carg)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         int                     ret = -EFAULT;
         nfs3_call_state_t       *cs = NULL;
+        nfs_user_t               nfu = {0, };
 
         if (!carg)
                 return ret;
@@ -4246,6 +4274,22 @@ nfs3_readdir_open_resume (void *carg)
                 goto nfs3err;
         }
 
+        /*
+         * NFS client will usually send us a readdirp without an opendir,
+         * which would cause us to skip our usual self-heal checks which occur
+         * in opendir for native protocol. To make sure those checks do happen,
+         * our most reliable option is to do our own opendir for any readdirp
+         * at the beginning of the directory.
+         */
+        if (cs->cookie == 0) {
+                nfs_request_user_init (&nfu, cs->req);
+                ret = nfs_opendir (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                                   nfs3svc_readdir_opendir_cbk, cs);
+                if (ret < 0) {
+                        gf_log (GF_NFS3, GF_LOG_ERROR, "auto-opendir failed");
+                }
+        }
+
         ret = nfs3_readdir_read_resume (cs);
         if (ret < 0)
                 stat = nfs3_errno_to_nfsstat3 (-ret);
@@ -4345,16 +4389,8 @@ nfs3svc_readdir (rpcsvc_request_t *req)
                 rpcsvc_request_seterr (req, GARBAGE_ARGS);
                 goto rpcerr;
         }
-
         cval = (uint64_t *) ra.cookieverf;
-
-        if (cval)
-                verf =  *cval;
-        else {
-                gf_log(GF_NFS3, GF_LOG_ERROR,
-                       "Error getting cookieverf from readdir args");
-                goto rpcerr;
-        }
+        verf =  *cval;
 
         ret = nfs3_readdir (req, &fh, ra.cookie, verf, ra.count, 0);
         if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
@@ -4385,16 +4421,8 @@ nfs3svc_readdirp (rpcsvc_request_t *req)
                 rpcsvc_request_seterr (req, GARBAGE_ARGS);
                 goto rpcerr;
         }
-
         cval = (uint64_t *) ra.cookieverf;
-
-        if (cval)
-                cverf = *cval;
-        else {
-                gf_log (GF_NFS3, GF_LOG_ERROR,
-                        "Error getting cookieverf from readdirp args");
-                goto rpcerr;
-	}
+        cverf = *cval;
 
         ret = nfs3_readdir (req, &fh, ra.cookie, cverf, ra.dircount,
                             ra.maxcount);
@@ -4426,7 +4454,8 @@ nfs3_fsstat_reply (rpcsvc_request_t *req, nfsstat3 stat, struct statvfs *fsbuf,
 
 int32_t
 nfs3_fsstat_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -4436,7 +4465,7 @@ nfs3_fsstat_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else
                 stat = NFS3_OK;
 
@@ -4450,7 +4479,8 @@ nfs3_fsstat_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int32_t
 nfs3_fsstat_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                        int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                        dict_t *xdata)
 {
         nfs_user_t              nfu = {0, };
         int                     ret = -EFAULT;
@@ -4463,7 +4493,7 @@ nfs3_fsstat_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
                 ret = -op_errno;
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
                 goto err;
         }
 
@@ -4610,7 +4640,8 @@ nfs3_fsinfo_reply (rpcsvc_request_t *req, nfsstat3 status, struct iatt *fsroot)
 
 int32_t
 nfs3svc_fsinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    dict_t *xdata)
 {
         nfsstat3                status = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -4621,7 +4652,7 @@ nfs3svc_fsinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_errno_to_nfsstat3 (op_errno);
+                status = nfs3_cbk_errno_status (op_ret, op_errno);
         }else
                 status = NFS3_OK;
 
@@ -4751,7 +4782,8 @@ nfs3_pathconf_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *buf)
 
 int32_t
 nfs3svc_pathconf_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      dict_t *xdata)
 {
         struct iatt             *sbuf = NULL;
         nfs3_call_state_t       *cs = NULL;
@@ -4762,7 +4794,7 @@ nfs3svc_pathconf_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else {
                 /* If stat fop failed, we can still send the other components
                  * in a pathconf reply.
@@ -4895,7 +4927,7 @@ nfs3_commit_reply (rpcsvc_request_t *req, nfsstat3 stat, uint64_t wverf,
 
 int32_t
 nfs3svc_commit_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         nfsstat3                stat = NFS3ERR_SERVERFAULT;
         nfs3_call_state_t       *cs = NULL;
@@ -4906,7 +4938,7 @@ nfs3svc_commit_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 gf_log (GF_NFS, GF_LOG_WARNING,
                         "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
                         cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_errno_to_nfsstat3 (op_errno);
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
         } else
                 stat = NFS3_OK;
 
@@ -5067,28 +5099,28 @@ rpcerr:
 
 
 rpcsvc_actor_t          nfs3svc_actors[NFS3_PROC_COUNT] = {
-        {"NULL",        NFS3_NULL,      nfs3svc_null,   NULL,   NULL, 0},
-        {"GETATTR",     NFS3_GETATTR,   nfs3svc_getattr,NULL,   NULL, 0},
-        {"SETATTR",     NFS3_SETATTR,   nfs3svc_setattr,NULL,   NULL, 0},
-        {"LOOKUP",      NFS3_LOOKUP,    nfs3svc_lookup, NULL,   NULL, 0},
-        {"ACCESS",      NFS3_ACCESS,    nfs3svc_access, NULL,   NULL, 0},
-        {"READLINK",    NFS3_READLINK,  nfs3svc_readlink,NULL,  NULL, 0},
-        {"READ",        NFS3_READ,      nfs3svc_read,   NULL,   NULL, 0},
-        {"WRITE",       NFS3_WRITE,     nfs3svc_write, nfs3svc_write_vec, nfs3svc_write_vecsizer, 0},
-        {"CREATE",      NFS3_CREATE,    nfs3svc_create, NULL,   NULL, 0},
-        {"MKDIR",       NFS3_MKDIR,     nfs3svc_mkdir,  NULL,   NULL, 0},
-        {"SYMLINK",     NFS3_SYMLINK,   nfs3svc_symlink,NULL,   NULL, 0},
-        {"MKNOD",       NFS3_MKNOD,     nfs3svc_mknod,  NULL,   NULL, 0},
-        {"REMOVE",      NFS3_REMOVE,    nfs3svc_remove, NULL,   NULL, 0},
-        {"RMDIR",       NFS3_RMDIR,     nfs3svc_rmdir,  NULL,   NULL, 0},
-        {"RENAME",      NFS3_RENAME,    nfs3svc_rename, NULL,   NULL, 0},
-        {"LINK",        NFS3_LINK,      nfs3svc_link,   NULL,   NULL, 0},
-        {"READDIR",     NFS3_READDIR,   nfs3svc_readdir,NULL,   NULL, 0},
-        {"READDIRPLUS", NFS3_READDIRP,  nfs3svc_readdirp,NULL,  NULL, 0},
-        {"FSSTAT",      NFS3_FSSTAT,    nfs3svc_fsstat, NULL,   NULL, 0},
-        {"FSINFO",      NFS3_FSINFO,    nfs3svc_fsinfo, NULL,   NULL, 0},
-        {"PATHCONF",    NFS3_PATHCONF,  nfs3svc_pathconf,NULL,  NULL, 0},
-        {"COMMIT",      NFS3_COMMIT,    nfs3svc_commit, NULL,   NULL, 0}
+        {"NULL",        NFS3_NULL,      nfs3svc_null,      NULL,   0, DRC_IDEMPOTENT},
+        {"GETATTR",     NFS3_GETATTR,   nfs3svc_getattr,   NULL,   0, DRC_IDEMPOTENT},
+        {"SETATTR",     NFS3_SETATTR,   nfs3svc_setattr,   NULL,   0, DRC_NON_IDEMPOTENT},
+        {"LOOKUP",      NFS3_LOOKUP,    nfs3svc_lookup,    NULL,   0, DRC_IDEMPOTENT},
+        {"ACCESS",      NFS3_ACCESS,    nfs3svc_access,    NULL,   0, DRC_IDEMPOTENT},
+        {"READLINK",    NFS3_READLINK,  nfs3svc_readlink,  NULL,   0, DRC_IDEMPOTENT},
+        {"READ",        NFS3_READ,      nfs3svc_read,      NULL,   0, DRC_IDEMPOTENT},
+        {"WRITE",       NFS3_WRITE,     nfs3svc_write,     nfs3svc_write_vecsizer, 0, DRC_NON_IDEMPOTENT},
+        {"CREATE",      NFS3_CREATE,    nfs3svc_create,    NULL,   0, DRC_NON_IDEMPOTENT},
+        {"MKDIR",       NFS3_MKDIR,     nfs3svc_mkdir,     NULL,   0, DRC_NON_IDEMPOTENT},
+        {"SYMLINK",     NFS3_SYMLINK,   nfs3svc_symlink,   NULL,   0, DRC_NON_IDEMPOTENT},
+        {"MKNOD",       NFS3_MKNOD,     nfs3svc_mknod,     NULL,   0, DRC_NON_IDEMPOTENT},
+        {"REMOVE",      NFS3_REMOVE,    nfs3svc_remove,    NULL,   0, DRC_NON_IDEMPOTENT},
+        {"RMDIR",       NFS3_RMDIR,     nfs3svc_rmdir,     NULL,   0, DRC_NON_IDEMPOTENT},
+        {"RENAME",      NFS3_RENAME,    nfs3svc_rename,    NULL,   0, DRC_NON_IDEMPOTENT},
+        {"LINK",        NFS3_LINK,      nfs3svc_link,      NULL,   0, DRC_NON_IDEMPOTENT},
+        {"READDIR",     NFS3_READDIR,   nfs3svc_readdir,   NULL,   0, DRC_IDEMPOTENT},
+        {"READDIRPLUS", NFS3_READDIRP,  nfs3svc_readdirp,  NULL,   0, DRC_IDEMPOTENT},
+        {"FSSTAT",      NFS3_FSSTAT,    nfs3svc_fsstat,    NULL,   0, DRC_IDEMPOTENT},
+        {"FSINFO",      NFS3_FSINFO,    nfs3svc_fsinfo,    NULL,   0, DRC_IDEMPOTENT},
+        {"PATHCONF",    NFS3_PATHCONF,  nfs3svc_pathconf,  NULL,   0, DRC_IDEMPOTENT},
+        {"COMMIT",      NFS3_COMMIT,    nfs3svc_commit,    NULL,   0, DRC_IDEMPOTENT}
 };
 
 
@@ -5105,21 +5137,48 @@ rpcsvc_program_t        nfs3prog = {
                         .min_auth       = AUTH_NULL,
 };
 
+/*
+ * This function rounds up the input value to multiple of 4096. Min and Max
+ * supported I/O size limits are 4KB (GF_NFS3_FILE_IO_SIZE_MIN) and
+ * 1MB (GF_NFS3_FILE_IO_SIZE_MAX).
+ */
+void
+nfs3_iosize_roundup_4KB (uint64_t *ioszptr)
+{
+        uint64_t iosize;
+        uint64_t iopages;
+
+        if (!ioszptr)
+                return;
+
+        iosize  = *ioszptr;
+        iopages = (iosize + GF_NFS3_IO_SIZE -1) >> GF_NFS3_IO_SHIFT;
+        iosize  = (iopages * GF_NFS3_IO_SIZE);
+
+        /* Double check - boundary conditions */
+        if (iosize < GF_NFS3_FILE_IO_SIZE_MIN) {
+                iosize = GF_NFS3_FILE_IO_SIZE_MIN;
+        } else if (iosize > GF_NFS3_FILE_IO_SIZE_MAX) {
+                iosize = GF_NFS3_FILE_IO_SIZE_MAX;
+        }
+
+        *ioszptr = iosize;
+}
 
 int
-nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
+nfs3_init_options (struct nfs3_state *nfs3, dict_t *options)
 {
         int      ret = -1;
         char     *optstr = NULL;
         uint64_t size64 = 0;
 
-        if ((!nfs3) || (!nfsx))
+        if ((!nfs3) || (!options))
                 return -1;
 
         /* nfs3.read-size */
         nfs3->readsize = GF_NFS3_RTPREF;
-        if (dict_get (nfsx->options, "nfs3.read-size")) {
-                ret = dict_get_str (nfsx->options, "nfs3.read-size", &optstr);
+        if (dict_get (options, "nfs3.read-size")) {
+                ret = dict_get_str (options, "nfs3.read-size", &optstr);
                 if (ret < 0) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
                                 " option: nfs3.read-size");
@@ -5128,19 +5187,21 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
                 }
 
                 ret = gf_string2bytesize (optstr, &size64);
-                nfs3->readsize = size64;
                 if (ret == -1) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
                                 " option: nfs3.read-size");
                         ret = -1;
                         goto err;
                 }
+
+                nfs3_iosize_roundup_4KB (&size64);
+                nfs3->readsize = size64;
         }
 
         /* nfs3.write-size */
         nfs3->writesize = GF_NFS3_WTPREF;
-        if (dict_get (nfsx->options, "nfs3.write-size")) {
-                ret = dict_get_str (nfsx->options, "nfs3.write-size", &optstr);
+        if (dict_get (options, "nfs3.write-size")) {
+                ret = dict_get_str (options, "nfs3.write-size", &optstr);
                 if (ret < 0) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
                                 " option: nfs3.write-size");
@@ -5149,19 +5210,21 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
                 }
 
                 ret = gf_string2bytesize (optstr, &size64);
-                nfs3->writesize = size64;
                 if (ret == -1) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
                                 " option: nfs3.write-size");
                         ret = -1;
                         goto err;
                 }
+
+                nfs3_iosize_roundup_4KB (&size64);
+                nfs3->writesize = size64;
         }
 
         /* nfs3.readdir.size */
         nfs3->readdirsize = GF_NFS3_DTPREF;
-        if (dict_get (nfsx->options, "nfs3.readdir-size")) {
-                ret = dict_get_str (nfsx->options,"nfs3.readdir-size", &optstr);
+        if (dict_get (options, "nfs3.readdir-size")) {
+                ret = dict_get_str (options,"nfs3.readdir-size", &optstr);
                 if (ret < 0) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read"
                                 " option: nfs3.readdir-size");
@@ -5170,15 +5233,16 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
                 }
 
                 ret = gf_string2bytesize (optstr, &size64);
-                nfs3->readdirsize = size64;
                 if (ret == -1) {
                         gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
                                 " option: nfs3.readdir-size");
                         ret = -1;
                         goto err;
                 }
-        }
 
+                nfs3_iosize_roundup_4KB (&size64);
+                nfs3->readdirsize = size64;
+        }
 
         /* We want to use the size of the biggest param for the io buffer size.
          */
@@ -5192,16 +5256,15 @@ nfs3_init_options (struct nfs3_state *nfs3, xlator_t *nfsx)
          * accommodate the NFS headers also in the same buffer. */
         nfs3->iobsize = nfs3->iobsize * 2;
 
-        /* mem-factor */
-        nfs3->memfactor = GF_NFS3_DEFAULT_MEMFACTOR;
         ret = 0;
 err:
         return ret;
 }
 
-
 int
-nfs3_init_subvolume_options (struct nfs3_state *nfs3, struct nfs3_export *exp)
+nfs3_init_subvolume_options (xlator_t *nfsx,
+                             struct nfs3_export *exp,
+                             dict_t *options)
 {
         int             ret = -1;
         char            *optstr = NULL;
@@ -5209,14 +5272,20 @@ nfs3_init_subvolume_options (struct nfs3_state *nfs3, struct nfs3_export *exp)
         char            *name = NULL;
         gf_boolean_t    boolt = _gf_false;
         uuid_t          volumeid = {0, };
-        dict_t          *options = NULL;
 
-        if ((!exp) || (!nfs3))
+        if ((!nfsx) || (!exp))
                 return -1;
 
-        options = nfs3->nfsx->options;
+        /* For init, fetch options from xlator but for
+         * reconfigure, take the parameter */
+        if (!options)
+                options = nfsx->options;
+
+        if (!options)
+                return (-1);
+
         uuid_clear (volumeid);
-        if (gf_nfs_dvm_off (nfs_state (nfs3->nfsx)))
+        if (gf_nfs_dvm_off (nfs_state (nfsx)))
                 goto no_dvm;
 
         ret = snprintf (searchkey, 1024, "nfs3.%s.volume-id",exp->subvol->name);
@@ -5382,7 +5451,7 @@ nfs3_init_subvolume (struct nfs3_state *nfs3, xlator_t *subvol)
         INIT_LIST_HEAD (&exp->explist);
         gf_log (GF_NFS3, GF_LOG_TRACE, "Initing state: %s", exp->subvol->name);
 
-        ret = nfs3_init_subvolume_options (nfs3, exp);
+        ret = nfs3_init_subvolume_options (nfs3->nfsx, exp, NULL);
         if (ret == -1) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init subvol");
                 goto exp_free;
@@ -5436,7 +5505,7 @@ nfs3_init_state (xlator_t *nfsx)
         unsigned int            localpool = 0;
         struct nfs_state        *nfs = NULL;
 
-        if (!nfsx)
+        if ((!nfsx) || (!nfsx->private))
                 return NULL;
 
         nfs3 = (struct nfs3_state *)GF_CALLOC (1, sizeof (*nfs3),
@@ -5447,7 +5516,7 @@ nfs3_init_state (xlator_t *nfsx)
         }
 
         nfs = nfsx->private;
-        ret = nfs3_init_options (nfs3, nfsx);
+        ret = nfs3_init_options (nfs3, nfsx->options);
         if (ret == -1) {
                 gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init options");
                 goto ret;
@@ -5455,7 +5524,7 @@ nfs3_init_state (xlator_t *nfsx)
 
         nfs3->iobpool = nfsx->ctx->iobuf_pool;
 
-        localpool = nfs3->memfactor * GF_NFS_CONCURRENT_OPS_MULT;
+        localpool = nfs->memfactor * GF_NFS_CONCURRENT_OPS_MULT;
         gf_log (GF_NFS3, GF_LOG_TRACE, "local pool: %d", localpool);
         nfs3->localpool = mem_pool_new (nfs3_call_state_t, localpool);
         if (!nfs3->localpool) {
@@ -5479,7 +5548,7 @@ nfs3_init_state (xlator_t *nfsx)
         LOCK_INIT (&nfs3->fdlrulock);
         nfs3->fdcount = 0;
 
-        rpcsvc_create_listeners (nfs->rpcsvc, nfsx->options, nfsx->name);
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, nfsx->options, nfsx->name);
         if (ret == -1) {
                 gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
                 goto free_localpool;
@@ -5521,4 +5590,39 @@ nfs3svc_init (xlator_t *nfsx)
         return &nfs3prog;
 }
 
+int
+nfs3_reconfigure_state (xlator_t *nfsx, dict_t *options)
+{
+        int                   ret   = -1;
+        struct nfs3_export    *exp  = NULL;
+        struct nfs_state      *nfs  = NULL;
+        struct nfs3_state     *nfs3 = NULL;
+
+        if ((!nfsx) || (!nfsx->private) || (!options))
+                goto out;
 
+        nfs = (struct nfs_state *)nfsx->private;
+        nfs3 = nfs->nfs3state;
+        if (!nfs3)
+                goto out;
+
+        ret = nfs3_init_options (nfs3, options);
+        if (ret) {
+                gf_log (GF_NFS3, GF_LOG_ERROR,
+                        "Failed to reconfigure options");
+                goto out;
+        }
+
+        list_for_each_entry (exp, &nfs3->exports, explist) {
+                ret = nfs3_init_subvolume_options (nfsx, exp, options);
+                if (ret) {
+                        gf_log (GF_NFS3, GF_LOG_ERROR,
+                                "Failed to reconfigure subvol options");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/nfs/server/src/nfs3.h b/xlators/nfs/server/src/nfs3.h
index 4a595e5ef..e64ef9d15 100644
--- a/xlators/nfs/server/src/nfs3.h
+++ b/xlators/nfs/server/src/nfs3.h
@@ -2,19 +2,10 @@
   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NFS3_H_
@@ -35,11 +26,11 @@
 #include "xdr-nfs3.h"
 #include "mem-pool.h"
 #include "nlm4.h"
-
+#include "acl3-xdr.h"
+#include "acl3.h"
 #include <sys/statvfs.h>
 
 #define GF_NFS3                 GF_NFS"-nfsv3"
-#define GF_NFS3_PORT            38467
 
 #define GF_NFS3_DEFAULT_MEMFACTOR       15
 #define GF_NFS3_IOBPOOL_MULT            GF_NFS_CONCURRENT_OPS_MULT
@@ -48,16 +39,39 @@
 
 
 /* Static values used for FSINFO
-FIXME: This should be configurable */
-#define GF_NFS3_RTMAX      (64 * GF_UNIT_KB)
-#define GF_NFS3_RTPREF     (64 * GF_UNIT_KB)
-#define GF_NFS3_RTMULT     (4 * GF_UNIT_KB)
-#define GF_NFS3_WTMAX      (64 * GF_UNIT_KB)
-#define GF_NFS3_WTPREF     (64 * GF_UNIT_KB)
-#define GF_NFS3_WTMULT     (4 * GF_UNIT_KB)
-#define GF_NFS3_DTMIN      (4 * GF_UNIT_KB)
-#define GF_NFS3_DTPREF     (64 * GF_UNIT_KB)
-#define GF_NFS3_MAXFILE    (1 * GF_UNIT_PB)
+ * To change the maximum rsize and wsize supported by the NFS client, adjust
+ * GF_NFS3_FILE_IO_SIZE_MAX. The Gluster NFS server defaults to 1MB(1048576)
+ * (same as kernel NFS server). For slower network, rsize/wsize can be trimmed
+ * to 16/32/64-KB. rsize and wsize can be tuned through nfs.read-size and
+ * nfs.write-size respectively.
+ *
+ * NB: For Kernel-NFS, NFS_MAX_FILE_IO_SIZE is 1048576U (1MB).
+ */
+#define GF_NFS3_FILE_IO_SIZE_MAX     (1  * GF_UNIT_MB) /* 1048576 */
+#define GF_NFS3_FILE_IO_SIZE_MIN     (4  * GF_UNIT_KB) /* 4096 */
+
+#define GF_NFS3_FILE_IO_SIZE_DEF     GF_NFS3_FILE_IO_SIZE_MAX
+
+#define GF_NFS3_RTMAX          GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_RTMIN          GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_RTPREF         GF_NFS3_FILE_IO_SIZE_DEF
+#define GF_NFS3_RTMULT         GF_NFS3_FILE_IO_SIZE_MIN
+
+#define GF_NFS3_WTMAX          GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_WTMIN          GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_WTPREF         GF_NFS3_FILE_IO_SIZE_DEF
+#define GF_NFS3_WTMULT         GF_NFS3_FILE_IO_SIZE_MIN
+
+/* This can be tuned through nfs.readdir-size */
+#define GF_NFS3_DTMAX          GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_DTMIN          GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_DTPREF         GF_NFS3_FILE_IO_SIZE_DEF
+
+#define GF_NFS3_MAXFILESIZE    (1 * GF_UNIT_PB)
+
+#define GF_NFS3_IO_SIZE        4096 /* 4-KB */
+#define GF_NFS3_IO_SHIFT       12   /* 2^12 = 4KB */
+
 /* FIXME: Handle time resolutions */
 #define GF_NFS3_TIMEDELTA_SECS     {1,0}
 #define GF_NFS3_TIMEDELTA_NSECS    {0,1}
@@ -120,20 +134,19 @@ typedef struct nfs3_state {
         uint64_t                serverstart;
 
         /* NFSv3 Protocol configurables */
-        size_t                  readsize;
-        size_t                  writesize;
-        size_t                  readdirsize;
+        uint64_t                readsize;
+        uint64_t                writesize;
+        uint64_t                readdirsize;
 
         /* Size of the iobufs used, depends on the sizes of the three params
          * above.
          */
-        size_t                  iobsize;
-
-        unsigned int            memfactor;
+        uint64_t                iobsize;
 
         struct list_head        fdlru;
         gf_lock_t               fdlrulock;
         int                     fdcount;
+        uint32_t                occ_logger;
 } nfs3_state_t;
 
 typedef enum nfs3_lookup_type {
@@ -155,6 +168,11 @@ typedef union args_ {
         nlm4_unlockargs nlm4_unlockargs;
         nlm4_shareargs nlm4_shareargs;
         nlm4_shareres nlm4_shareres;
+        nlm4_freeallargs nlm4_freeallargs;
+        getaclargs getaclargs;
+        setaclargs setaclargs;
+        getaclreply getaclreply;
+        setaclreply setaclreply;
 } args;
 
 
@@ -231,8 +249,17 @@ struct nfs3_local {
         nlm4_lkowner_t          lkowner;
         char                    cookiebytes[1024];
         struct nfs3_fh          lockfh;
+        int                     monitor;
         rpc_transport_t         *trans;
         call_frame_t            *frame;
+
+        /* ACL */
+        aclentry                aclentry[NFS_ACL_MAX_ENTRIES];
+        aclentry                daclentry[NFS_ACL_MAX_ENTRIES];
+        int                     aclcount;
+        char                    aclxattr[NFS_ACL_MAX_ENTRIES*8 + 4];
+        int                     daclcount;
+        char                    daclxattr[NFS_ACL_MAX_ENTRIES*8 + 4];
 };
 
 #define nfs3_is_revalidate_lookup(cst) ((cst)->lookuptype == GF_NFS3_REVALIDATE)
@@ -248,9 +275,13 @@ struct inode_op_queue {
         pthread_mutex_t         qlock;
 };
 
+extern rpcsvc_program_t *
+nfs3svc_init (xlator_t *nfsx);
 
+extern int
+nfs3_reconfigure_state (xlator_t *nfsx, dict_t *options);
 
+extern uint64_t
+nfs3_request_xlator_deviceid (rpcsvc_request_t *req);
 
-extern rpcsvc_program_t *
-nfs3svc_init (xlator_t *nfsx);
 #endif
diff --git a/xlators/nfs/server/src/nlm4.c b/xlators/nfs/server/src/nlm4.c
index 170c994d5..c186537ea 100644
--- a/xlators/nfs/server/src/nlm4.c
+++ b/xlators/nfs/server/src/nlm4.c
@@ -2,19 +2,10 @@
   Copyright (c) 2012 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -43,10 +34,12 @@
 #include "rpc-clnt.h"
 #include "nsm-xdr.h"
 #include "nlmcbk-xdr.h"
+#include "run.h"
 #include <unistd.h>
 #include <rpc/pmap_clnt.h>
 #include <rpc/rpc.h>
 #include <rpc/xdr.h>
+#include <statedump.h>
 
 /* TODO:
  * 1) 2 opens racing .. creating an fd leak.
@@ -101,12 +94,23 @@ nfs3_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
 xlator_t *
 nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh);
 
-#define nlm4_map_fh_to_volume(nfs3state, handle, rqst, volume, status, label) \
+#define nlm4_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
         do {                                                            \
-                volume = nfs3_fh_to_xlator ((nfs3state), handle);       \
+                char exportid[256], gfid[256];                          \
+                rpc_transport_t *trans = NULL;                          \
+                volume = nfs3_fh_to_xlator ((nfs3state), &handle);      \
                 if (!volume) {                                          \
+                        uuid_unparse (handle.exportid, exportid);       \
+                        uuid_unparse (handle.gfid, gfid);               \
+                        trans = rpcsvc_request_transport (req);         \
                         gf_log (GF_NLM, GF_LOG_ERROR, "Failed to map "  \
-                                "FH to vol");                           \
+                                "FH to vol: client=%s, exportid=%s, gfid=%s",\
+                                trans->peerinfo.identifier, exportid,   \
+                                gfid);                                  \
+                        gf_log (GF_NLM, GF_LOG_ERROR,                   \
+                                "Stale nfs client %s must be trying to "\
+                                "connect to a deleted volume, please "  \
+                                "unmount it.", trans->peerinfo.identifier);\
                         status = nlm4_stale_fh;                         \
                         goto label;                                     \
                 } else {                                                \
@@ -126,10 +130,23 @@ nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh);
               }                                                         \
         } while (0)                                                     \
 
-#define nlm4_check_fh_resolve_status(cst, _stat, erlabl)               \
+#define nlm4_check_fh_resolve_status(cst, nfstat, erlabl)               \
         do {                                                            \
+                xlator_t *xlatorp = NULL;                               \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
                 if ((cst)->resolve_ret < 0) {                           \
-                        _stat = nlm4_errno_to_nlm4stat (cst->resolve_errno);\
+                        trans = rpcsvc_request_transport (cst->req);    \
+                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
+                                                     &cst->resolvefh);  \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,           \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid);                                \
+                        gf_log (GF_NLM, GF_LOG_ERROR, "Unable to resolve FH"\
+                                ": %s", buf);                           \
+                        nfstat = nlm4_errno_to_nlm4stat (cst->resolve_errno);\
                         goto erlabl;                                    \
                 }                                                       \
         } while (0)                                                     \
@@ -176,14 +193,42 @@ nlm4_prep_nlm4_unlockargs (nlm4_unlockargs *args, struct nfs3_fh *fh,
 }
 
 void
+nlm4_prep_shareargs (nlm4_shareargs *args, struct nfs3_fh *fh,
+                     nlm4_lkowner_t *oh, char *cookiebytes)
+{
+        memset (args, 0, sizeof (*args));
+        args->share.fh.n_bytes = (void *)fh;
+        args->share.oh.n_bytes = (void *)oh;
+        args->cookie.n_bytes = (void *)cookiebytes;
+}
+
+void
+nlm4_prep_freeallargs (nlm4_freeallargs *args, nlm4_lkowner_t *oh)
+{
+        memset (args, 0, sizeof (*args));
+        args->name = (void *)oh;
+}
+
+void
 nlm_copy_lkowner (gf_lkowner_t *dst, netobj *src)
 {
         dst->len = src->n_len;
         memcpy (dst->data, src->n_bytes, dst->len);
 }
 
+int
+nlm_is_oh_same_lkowner (gf_lkowner_t *a, netobj *b)
+{
+        if (!a || !b) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "invalid args");
+                return -1;
+        }
+
+        return (a->len == b->n_len &&
+                !memcmp (a->data, b->n_bytes, a->len));
+}
 
-nfsstat3
+nlm4_stats
 nlm4_errno_to_nlm4stat (int errnum)
 {
         nlm4_stats        stat = nlm4_denied;
@@ -228,10 +273,34 @@ nlm4_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req)
         cs->req = req;
         cs->nfsx = s->nfsx;
         cs->nfs3state = s;
+        cs->monitor = 1;
 
         return cs;
 }
 
+int
+nlm_monitor (char *caller_name)
+{
+        nlm_client_t *nlmclnt = NULL;
+        int monitor = -1;
+
+        LOCK (&nlm_client_list_lk);
+        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
+                if (!strcmp(caller_name, nlmclnt->caller_name)) {
+                        monitor = nlmclnt->nsm_monitor;
+                        nlmclnt->nsm_monitor = 1;
+                        break;
+                }
+        }
+        UNLOCK (&nlm_client_list_lk);
+
+        if (monitor == -1)
+                gf_log (GF_NLM, GF_LOG_ERROR, "%s was not found in "
+                        "the nlmclnt list", caller_name);
+
+        return monitor;
+}
+
 rpc_clnt_t *
 nlm_get_rpc_clnt (char *caller_name)
 {
@@ -261,8 +330,6 @@ nlm_set_rpc_clnt (rpc_clnt_t *rpc_clnt, char *caller_name)
         nlm_client_t *nlmclnt = NULL;
         int nlmclnt_found = 0;
         int ret = -1;
-        rpc_clnt_t *rpc_clnt_old = NULL;
-        char *old_name = NULL;
 
         LOCK (&nlm_client_list_lk);
         list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
@@ -271,32 +338,27 @@ nlm_set_rpc_clnt (rpc_clnt_t *rpc_clnt, char *caller_name)
                         break;
                 }
         }
+
         if (!nlmclnt_found) {
                 nlmclnt = GF_CALLOC (1, sizeof(*nlmclnt),
                                      gf_nfs_mt_nlm4_nlmclnt);
-                if (nlmclnt == NULL) {
-                        gf_log (GF_NLM, GF_LOG_DEBUG, "malloc error");
+                if (nlmclnt == NULL)
                         goto ret;
-                }
 
                 INIT_LIST_HEAD(&nlmclnt->fdes);
                 INIT_LIST_HEAD(&nlmclnt->nlm_clients);
+                INIT_LIST_HEAD(&nlmclnt->shares);
 
                 list_add (&nlmclnt->nlm_clients, &nlm_client_list);
+                nlmclnt->caller_name = gf_strdup (caller_name);
         }
-        rpc_clnt_old = nlmclnt->rpc_clnt;
-        old_name = nlmclnt->caller_name;
-        if (rpc_clnt)
-                nlmclnt->rpc_clnt = rpc_clnt_ref (rpc_clnt);
-        nlmclnt->caller_name = gf_strdup (caller_name);
 
+        if (nlmclnt->rpc_clnt == NULL) {
+                nlmclnt->rpc_clnt = rpc_clnt_ref (rpc_clnt);
+        }
         ret = 0;
 ret:
         UNLOCK (&nlm_client_list_lk);
-        if (rpc_clnt_old)
-                rpc_clnt_unref (rpc_clnt_old);
-        if (old_name)
-                GF_FREE (old_name);
         return ret;
 }
 
@@ -318,8 +380,12 @@ nlm_unset_rpc_clnt (rpc_clnt_t *rpc)
         if (rpc_clnt == NULL) {
                 return -1;
         }
-        if (rpc_clnt)
+        if (rpc_clnt) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&rpc_clnt->conn);
+
                 rpc_clnt_unref (rpc_clnt);
+        }
         return 0;
 }
 
@@ -347,6 +413,7 @@ nlm_add_nlmclnt (char *caller_name)
 
                 INIT_LIST_HEAD(&nlmclnt->fdes);
                 INIT_LIST_HEAD(&nlmclnt->nlm_clients);
+                INIT_LIST_HEAD(&nlmclnt->shares);
 
                 list_add (&nlmclnt->nlm_clients, &nlm_client_list);
                 nlmclnt->caller_name = gf_strdup (caller_name);
@@ -360,10 +427,11 @@ ret:
 int
 nlm4svc_submit_reply (rpcsvc_request_t *req, void *arg, nlm4_serializer sfunc)
 {
-        struct iovec            outmsg = {0, };
-        struct iobuf            *iob = NULL;
-        struct nfs3_state       *nfs3 = NULL;
-        int                     ret = -1;
+        struct iovec            outmsg  = {0, };
+        struct iobuf            *iob    = NULL;
+        struct nfs3_state       *nfs3   = NULL;
+        int                     ret     = -1;
+        ssize_t                 msglen  = 0;
         struct iobref           *iobref = NULL;
 
         if (!req)
@@ -388,7 +456,12 @@ nlm4svc_submit_reply (rpcsvc_request_t *req, void *arg, nlm4_serializer sfunc)
         /* Use the given serializer to translate the give C structure in arg
          * to XDR format which will be written into the buffer in outmsg.
          */
-        outmsg.iov_len = sfunc (outmsg, arg);
+        msglen = sfunc (outmsg, arg);
+        if (msglen < 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+        outmsg.iov_len = msglen;
 
         iobref = iobref_new ();
         if (iobref == NULL) {
@@ -396,12 +469,14 @@ nlm4svc_submit_reply (rpcsvc_request_t *req, void *arg, nlm4_serializer sfunc)
                 goto ret;
         }
 
-        iobref_add (iobref, iob);
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
 
         /* Then, submit the message for transmission. */
         ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-        iobuf_unref (iob);
-        iobref_unref (iobref);
         if (ret == -1) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "Reply submission failed");
                 goto ret;
@@ -409,6 +484,11 @@ nlm4svc_submit_reply (rpcsvc_request_t *req, void *arg, nlm4_serializer sfunc)
 
         ret = 0;
 ret:
+        if (iob)
+                iobuf_unref (iob);
+        if (iobref)
+                iobref_unref (iobref);
+
         return ret;
 }
 
@@ -416,7 +496,7 @@ typedef int (*nlm4_resume_fn_t) (void *cs);
 
 int32_t
 nlm4_file_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd)
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         nfs3_call_state_t *cs = frame->local;
 
@@ -429,15 +509,17 @@ nlm4_file_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-int nsm_monitor(char *host)
+void *
+nsm_monitor(void *arg)
 {
         CLIENT *clnt = NULL;
         enum clnt_stat ret;
         struct mon nsm_mon;
         struct sm_stat_res res;
         struct timeval tout = { 5, 0 };
-        int retstat = -1;
+        char *host = NULL;
 
+        host = arg;
         nsm_mon.mon_id.mon_name = gf_strdup(host);
         nsm_mon.mon_id.my_id.my_name = gf_strdup("localhost");
         nsm_mon.mon_id.my_id.my_prog = NLMCBK_PROGRAM;
@@ -452,7 +534,8 @@ int nsm_monitor(char *host)
         clnt = clnt_create("localhost", SM_PROG, SM_VERS, "tcp");
         if(!clnt)
         {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Clnt_create()");
+                gf_log (GF_NLM, GF_LOG_ERROR, "%s",
+                        clnt_spcreateerror ("Clnt_create()"));
                 goto out;
         }
 
@@ -471,32 +554,41 @@ int nsm_monitor(char *host)
                         clnt_sperrno(ret));
                 goto out;
         }
-        retstat = 0;
+
 out:
         GF_FREE(nsm_mon.mon_id.mon_name);
         GF_FREE(nsm_mon.mon_id.my_id.my_name);
-        clnt_destroy(clnt);
-        return retstat;
+        if (clnt != NULL)
+                clnt_destroy(clnt);
+        return NULL;
 }
 
 nlm_client_t *
-nlm_get_uniq (char *caller_name)
+__nlm_get_uniq (char *caller_name)
 {
         nlm_client_t *nlmclnt = NULL;
-        int nlmclnt_found = 0;
 
-        LOCK (&nlm_client_list_lk);
+        if (!caller_name)
+                return NULL;
+
         list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        nlmclnt_found = 1;
-                        break;
-                }
+                if (!strcmp(caller_name, nlmclnt->caller_name))
+                        return nlmclnt;
         }
+
+        return NULL;
+}
+
+nlm_client_t *
+nlm_get_uniq (char *caller_name)
+{
+        nlm_client_t *nlmclnt = NULL;
+
+        LOCK (&nlm_client_list_lk);
+        nlmclnt = __nlm_get_uniq (caller_name);
         UNLOCK (&nlm_client_list_lk);
-        if (nlmclnt_found)
-                return nlmclnt;
-        else
-                return NULL;
+
+        return nlmclnt;
 }
 
 
@@ -505,15 +597,19 @@ nlm4_file_open_and_resume(nfs3_call_state_t *cs, nlm4_resume_fn_t resume)
 {
         fd_t *fd = NULL;
         int ret = -1;
+        int flags = 0;
         nlm_client_t *nlmclnt = NULL;
         call_frame_t *frame = NULL;
 
+        if (cs->args.nlm4_lockargs.exclusive == _gf_false)
+                flags = O_RDONLY;
+        else
+                flags = O_WRONLY;
+
         nlmclnt = nlm_get_uniq (cs->args.nlm4_lockargs.alock.caller_name);
         if (nlmclnt == NULL) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
-                cs->resolve_ret = -1;
-                cs->resume_fn(cs);
-                ret = -1;
+                ret = -ENOLCK;
                 goto err;
         }
         cs->resume_fn = resume;
@@ -529,22 +625,28 @@ nlm4_file_open_and_resume(nfs3_call_state_t *cs, nlm4_resume_fn_t resume)
         fd = fd_create_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
         if (fd == NULL) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "fd_create_uint64() returned NULL");
-                cs->resolve_ret = -1;
-                cs->resume_fn(cs);
-                ret = -1;
+                ret = -ENOLCK;
                 goto err;
         }
 
         cs->fd = fd;
 
         frame = create_frame (cs->nfsx, cs->nfsx->ctx->pool);
+        if (!frame) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to create frame");
+                ret = -ENOMEM;
+                goto err;
+        }
+
         frame->root->pid = NFS_PID;
-        frame->root->uid = 0;
-        frame->root->gid = 0;
+        frame->root->uid = rpcsvc_request_uid (cs->req);
+        frame->root->gid = rpcsvc_request_gid (cs->req);
         frame->local = cs;
-        STACK_WIND_COOKIE(frame, nlm4_file_open_cbk, cs->nfsx, cs->nfsx,
-                          cs->nfsx->fops->open, &cs->resolvedloc, O_RDWR,
-                          cs->fd, GF_OPEN_NOWB);
+        nfs_fix_groups (cs->nfsx, frame->root);
+
+        STACK_WIND_COOKIE (frame, nlm4_file_open_cbk, cs->vol, cs->vol,
+                          cs->vol->fops->open, &cs->resolvedloc, flags,
+                          cs->fd, NULL);
         ret = 0;
 err:
         return ret;
@@ -645,7 +747,8 @@ nlm4_test_reply (nfs3_call_state_t *cs, nlm4_stats stat, struct gf_flock *flock)
 
 int
 nlm4svc_test_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct gf_flock *flock)
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                  dict_t *xdata)
 {
         nlm4_stats                      stat = nlm4_denied;
         nfs3_call_state_t              *cs = NULL;
@@ -666,7 +769,6 @@ err:
 int
 nlm4_test_fd_resume (void *carg)
 {
-        nlm4_stats                      stat = nlm4_denied;
         int                             ret = -EFAULT;
         nfs_user_t                      nfu = {0, };
         nfs3_call_state_t               *cs = NULL;
@@ -676,21 +778,12 @@ nlm4_test_fd_resume (void *carg)
                 return ret;
 
         cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
         nfs_request_user_init (&nfu, cs->req);
         nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_testargs.alock,
                                cs->args.nlm4_testargs.exclusive);
         nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_testargs.alock.oh);
         ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_GETLK, &flock,
                       nlm4svc_test_cbk, cs);
-        if (ret < 0)
-                stat = nlm4_errno_to_nlm4stat (-ret);
-nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Unable to call lk()");
-                nlm4_test_reply (cs, stat, &flock);
-                nfs3_call_state_wipe (cs);
-        }
 
         return ret;
 }
@@ -710,14 +803,15 @@ nlm4_test_resume (void *carg)
         cs = (nfs3_call_state_t *)carg;
         nlm4_check_fh_resolve_status (cs, stat, nlm4err);
         fd = fd_anonymous (cs->resolvedloc.inode);
+        if (!fd)
+                goto nlm4err;
         cs->fd = fd;
         ret = nlm4_test_fd_resume (cs);
-        if (ret < 0)
-                stat = nlm4_errno_to_nlm4stat (-ret);
 
 nlm4err:
         if (ret < 0) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "unable to open_and_resume");
+                stat = nlm4_errno_to_nlm4stat (-ret);
                 nlm4_test_reply (cs, stat, NULL);
                 nfs3_call_state_wipe (cs);
         }
@@ -753,7 +847,7 @@ nlm4svc_test (rpcsvc_request_t *req)
         }
 
         nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, &fh, req, vol, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
 
         if (nlm_grace_period) {
                 gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
@@ -778,9 +872,9 @@ nlm4err:
         }
 
 rpcerr:
-        if (ret < 0) {
+        if (ret < 0)
                 nfs3_call_state_wipe (cs);
-        }
+
         return ret;
 }
 
@@ -792,53 +886,67 @@ nlm4svc_send_granted_cbk (struct rpc_req *req, struct iovec *iov, int count,
         return 0;
 }
 
-int nlm_rpcclnt_notify (struct rpc_clnt *rpc, void *mydata,
-                        rpc_clnt_event_t fn, void *data)
+void
+nlm4svc_send_granted (nfs3_call_state_t *cs);
+
+int
+nlm_rpcclnt_notify (struct rpc_clnt *rpc_clnt, void *mydata,
+                    rpc_clnt_event_t fn, void *data)
 {
-        nlm_condmutex_t *cm = NULL;
-        int ret;
-        cm = mydata;
+        int                ret         = 0;
+        char              *caller_name = NULL;
+        nfs3_call_state_t *cs          = NULL;
+
+        cs = mydata;
+        caller_name = cs->args.nlm4_lockargs.alock.caller_name;
+
         switch (fn) {
         case RPC_CLNT_CONNECT:
-                ret = pthread_cond_broadcast (&cm->cond);
-                if (ret!=0)
-                        gf_log (GF_NLM, GF_LOG_ERROR, "cond_broadcast error %s",
-                                strerror (errno));
+                ret = nlm_set_rpc_clnt (rpc_clnt, caller_name);
+                if (ret == -1) {
+                        gf_log (GF_NLM, GF_LOG_ERROR, "Failed to set rpc clnt");
+                        goto err;
+                }
+                rpc_clnt_unref (rpc_clnt);
+                nlm4svc_send_granted (cs);
+
                 break;
+
         case RPC_CLNT_MSG:
                 break;
+
         case RPC_CLNT_DISCONNECT:
-                nlm_unset_rpc_clnt(rpc);
+                nlm_unset_rpc_clnt (rpc_clnt);
+                break;
+        default:
                 break;
         }
+
+ err:
         return 0;
 }
 
-void
-nlm4svc_send_granted (nfs3_call_state_t *cs);
-
 void *
 nlm4_establish_callback (void *csarg)
 {
-        nfs3_call_state_t *cs = NULL;
-        struct sockaddr_storage sa;
-        struct sockaddr *sockaddr = NULL;
-        dict_t *options = NULL;
-        char peerip[INET6_ADDRSTRLEN+1], *portstr = NULL;
-        rpc_clnt_t *rpc_clnt = NULL;
-        int port = -1;
-        int ret = -1;
-        char *caller_name = NULL;
+        int                           ret                        = -1;
+        nfs3_call_state_t            *cs                         = NULL;
+        union gf_sock_union           sock_union;
+        dict_t                       *options                    = NULL;
+        char                          peerip[INET6_ADDRSTRLEN+1] = {0};
+        char                         *portstr                    = NULL;
+        char                          myip[INET6_ADDRSTRLEN+1]   = {0};
+        rpc_clnt_t                   *rpc_clnt                   = NULL;
+        int                           port                       = -1;
+
 
         cs = (nfs3_call_state_t *) csarg;
         glusterfs_this_set (cs->nfsx);
 
-        caller_name = cs->args.nlm4_lockargs.alock.caller_name;
-        nsm_monitor (caller_name);
+        rpc_transport_get_peeraddr (cs->trans, NULL, 0, &sock_union.storage,
+                                    sizeof (sock_union.storage));
 
-        rpc_transport_get_peeraddr (cs->trans, NULL, 0, &sa, sizeof (sa));
-        sockaddr = (struct sockaddr*) &sa;
-        switch (sockaddr->sa_family) {
+        switch (sock_union.sa.sa_family) {
         case AF_INET6:
         /* can not come here as NLM listens on IPv4 */
                 gf_log (GF_NLM, GF_LOG_ERROR, "NLM is not supported on IPv6 in"
@@ -851,9 +959,11 @@ nlm4_establish_callback (void *csarg)
                 break;
 */
         case AF_INET:
-                inet_ntop (AF_INET,
-                           &((struct sockaddr_in *)sockaddr)->sin_addr,
-                           peerip, INET6_ADDRSTRLEN+1);
+                inet_ntop (AF_INET, &sock_union.sin.sin_addr, peerip,
+                           INET6_ADDRSTRLEN+1);
+                inet_ntop (AF_INET, &(((struct sockaddr_in *)&cs->trans->myinfo.sockaddr)->sin_addr),
+                           myip, INET6_ADDRSTRLEN + 1);
+
                 break;
         default:
                 break;
@@ -861,7 +971,7 @@ nlm4_establish_callback (void *csarg)
         }
 
         /* looks like libc rpc supports only ipv4 */
-        port = pmap_getport ((struct sockaddr_in*)sockaddr, NLM_PROGRAM,
+        port = pmap_getport (&sock_union.sin, NLM_PROGRAM,
                              NLM_V4, IPPROTO_TCP);
 
         if (port == 0) {
@@ -893,6 +1003,13 @@ nlm4_establish_callback (void *csarg)
                 gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_dynstr error");
                 goto err;
         }
+
+        /* needed in case virtual IP is used */
+        ret = dict_set_dynstr (options, "transport.socket.source-addr",
+                               gf_strdup (myip));
+        if (ret == -1)
+                goto err;
+
         ret = dict_set_str (options, "auth-null", "on");
         if (ret == -1) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_dynstr error");
@@ -905,32 +1022,25 @@ nlm4_establish_callback (void *csarg)
                 gf_log (GF_NLM, GF_LOG_ERROR, "rpc_clnt NULL");
                 goto err;
         }
-        nlm_condmutex_t *cm;
-        cm = GF_CALLOC (1, sizeof(*cm), gf_nfs_mt_nlm4_cm);
-        pthread_mutex_init (&cm->mutex, NULL);
-        pthread_cond_init (&cm->cond, NULL);
-        ret = rpc_clnt_register_notify (rpc_clnt, nlm_rpcclnt_notify,
-                                        cm);
+
+        ret = rpc_clnt_register_notify (rpc_clnt, nlm_rpcclnt_notify, cs);
         if (ret == -1) {
                 gf_log (GF_NLM, GF_LOG_ERROR,"rpc_clnt_register_connect error");
                 goto err;
         }
+
+        /* After this connect succeeds, granted msg is sent in notify */
         ret = rpc_transport_connect (rpc_clnt->conn.trans, port);
-        pthread_cond_wait (&cm->cond, &cm->mutex);
-        pthread_mutex_destroy (&cm->mutex);
-        GF_FREE (cm);
-        rpc_clnt_set_connected (&rpc_clnt->conn);
-        ret = nlm_set_rpc_clnt (rpc_clnt, caller_name);
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_ptr error");
-                goto err;
-        }
-        nlm4svc_send_granted (cs);
+
+        if (ret == -1 && EINPROGRESS == errno)
+                ret = 0;
+
 err:
-        if (rpc_clnt) {
+        if (ret == -1 && rpc_clnt) {
                 rpc_clnt_unref (rpc_clnt);
         }
-        return NULL;
+
+        return rpc_clnt;
 }
 
 void
@@ -943,32 +1053,28 @@ nlm4svc_send_granted (nfs3_call_state_t *cs)
         struct iobuf *iobuf = NULL;
         struct iobref *iobref = NULL;
         char peerip[INET6_ADDRSTRLEN+1];
-        pthread_t thr;
-        struct sockaddr_storage sa;
-        struct sockaddr *sockaddr = NULL;
+        union gf_sock_union sock_union;
 
+        rpc_clnt = nlm_get_rpc_clnt (cs->args.nlm4_lockargs.alock.caller_name);
+        if (rpc_clnt == NULL) {
+                nlm4_establish_callback ((void*)cs);
+                return;
+        }
 
-        rpc_transport_get_peeraddr (cs->trans, NULL, 0, &sa, sizeof (sa));
-        sockaddr = (struct sockaddr*) &sa;
-        switch (sockaddr->sa_family) {
+        rpc_transport_get_peeraddr (cs->trans, NULL, 0, &sock_union.storage,
+                                    sizeof (sock_union.storage));
+
+        switch (sock_union.sa.sa_family) {
         case AF_INET6:
-                inet_ntop (AF_INET6,
-                           &((struct sockaddr_in6 *)sockaddr)->sin6_addr,
-                           peerip, INET6_ADDRSTRLEN+1);
+                inet_ntop (AF_INET6, &sock_union.sin6.sin6_addr, peerip,
+                           INET6_ADDRSTRLEN+1);
                 break;
         case AF_INET:
-                inet_ntop (AF_INET,
-                           &((struct sockaddr_in *)sockaddr)->sin_addr,
-                           peerip, INET6_ADDRSTRLEN+1);
+                inet_ntop (AF_INET, &sock_union.sin.sin_addr, peerip,
+                           INET6_ADDRSTRLEN+1);
+                break;
         default:
                 break;
-                /* FIXME: handle the error */
-        }
-
-        rpc_clnt = nlm_get_rpc_clnt (cs->args.nlm4_lockargs.alock.caller_name);
-        if (rpc_clnt == NULL) {
-                pthread_create (&thr, NULL, nlm4_establish_callback, (void*)cs);
-                return;
         }
 
         testargs.cookie = cs->args.nlm4_lockargs.cookie;
@@ -993,13 +1099,15 @@ nlm4svc_send_granted (nfs3_call_state_t *cs)
                 goto ret;
         }
 
-        iobref_add (iobref, iobuf);
+        ret = iobref_add (iobref, iobuf);
+        if (ret) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
 
         ret = rpc_clnt_submit (rpc_clnt, &nlm4clntprog, NLM4_GRANTED,
-                               nlm4svc_send_granted_cbk,
-                               &outmsg, 1,
-                               NULL,
-                               0, iobref, cs->frame, NULL, 0,
+                               nlm4svc_send_granted_cbk, &outmsg, 1,
+                               NULL, 0, iobref, cs->frame, NULL, 0,
                                NULL, 0, NULL);
 
         if (ret < 0) {
@@ -1007,6 +1115,11 @@ nlm4svc_send_granted (nfs3_call_state_t *cs)
                 goto ret;
         }
 ret:
+        if (iobref)
+                iobref_unref (iobref);
+        if (iobuf)
+                iobuf_unref (iobuf);
+
         rpc_clnt_unref (rpc_clnt);
         nfs3_call_state_wipe (cs);
         return;
@@ -1178,28 +1291,36 @@ ret:
 
 int
 nlm4svc_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct gf_flock *flock)
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                  dict_t *xdata)
 {
-        nlm4_stats                      stat = nlm4_denied;
-        nfs3_call_state_t              *cs = NULL;
-        int                             transit_cnt = -1;
+        nlm4_stats                       stat        = nlm4_denied;
+        int                              transit_cnt = -1;
+        char                            *caller_name = NULL;
+        nfs3_call_state_t               *cs          = NULL;
+        pthread_t                        thr;
 
         cs = frame->local;
+        caller_name = cs->args.nlm4_lockargs.alock.caller_name;
+        transit_cnt = nlm_dec_transit_count (cs->fd, caller_name);
 
-        transit_cnt = nlm_dec_transit_count (cs->fd,
-                                             cs->args.nlm4_lockargs.alock.caller_name);
         if (op_ret == -1) {
                 if (transit_cnt == 0)
-                        nlm_search_and_delete (cs->fd,
-                                               cs->args.nlm4_lockargs.alock.caller_name);
+                        nlm_search_and_delete (cs->fd, caller_name);
                 stat = nlm4_errno_to_nlm4stat (op_errno);
                 goto err;
-        } else
+        } else {
                 stat = nlm4_granted;
+                if (cs->monitor && !nlm_monitor (caller_name)) {
+                        /* FIXME: handle nsm_monitor failure */
+                        pthread_create (&thr, NULL, nsm_monitor, (void*)caller_name);
+                }
+        }
 
 err:
         if (cs->args.nlm4_lockargs.block) {
                 cs->frame = copy_frame (frame);
+                frame->local = NULL;
                 nlm4svc_send_granted (cs);
         } else {
                 nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
@@ -1223,7 +1344,6 @@ nlm4_lock_fd_resume (void *carg)
 
         cs = (nfs3_call_state_t *)carg;
         nlm4_check_fh_resolve_status (cs, stat, nlm4err);
-
         (void) nlm_search_and_add (cs->fd,
                                    cs->args.nlm4_lockargs.alock.caller_name);
         nfs_request_user_init (&nfu, cs->req);
@@ -1235,14 +1355,17 @@ nlm4_lock_fd_resume (void *carg)
                                     nlm4_blocked);
                 ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLKW,
                               &flock, nlm4svc_lock_cbk, cs);
+                /* FIXME: handle error from nfs_lk() specially  by just
+                 * cleaning up cs and unblock the client lock request.
+                 */
+                ret = 0;
         } else
                 ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
                               &flock, nlm4svc_lock_cbk, cs);
 
-        if (ret < 0)
-                stat = nlm4_errno_to_nlm4stat (-ret);
 nlm4err:
         if (ret < 0) {
+                stat = nlm4_errno_to_nlm4stat (-ret);
                 gf_log (GF_NLM, GF_LOG_ERROR, "unable to call lk()");
                 nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
                                     stat);
@@ -1257,8 +1380,8 @@ int
 nlm4_lock_resume (void *carg)
 {
         nlm4_stats                      stat = nlm4_failed;
-        int                             ret = -1;
-        nfs3_call_state_t               *cs = NULL;
+        int                             ret  = -1;
+        nfs3_call_state_t              *cs   = NULL;
 
         if (!carg)
                 return ret;
@@ -1266,12 +1389,11 @@ nlm4_lock_resume (void *carg)
         cs = (nfs3_call_state_t *)carg;
         nlm4_check_fh_resolve_status (cs, stat, nlm4err);
         ret = nlm4_file_open_and_resume (cs, nlm4_lock_fd_resume);
-        if (ret < 0)
-                stat = nlm4_errno_to_nlm4stat (-ret);
 
 nlm4err:
         if (ret < 0) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "unable to open and resume");
+                stat = nlm4_errno_to_nlm4stat (-ret);
                 nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
                                     stat);
                 nfs3_call_state_wipe (cs);
@@ -1280,17 +1402,16 @@ nlm4err:
         return ret;
 }
 
-
 int
-nlm4svc_lock (rpcsvc_request_t *req)
+nlm4svc_lock_common (rpcsvc_request_t *req, int mon)
 {
-        xlator_t                        *vol = NULL;
-        nlm4_stats                      stat = nlm4_failed;
-        struct nfs_state               *nfs = NULL;
-        nfs3_state_t                   *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-        int                             ret = RPCSVC_ACTOR_ERROR;
-        struct nfs3_fh                  fh = {{0}, };
+        int                              ret      = RPCSVC_ACTOR_ERROR;
+        nlm4_stats                       stat     = nlm4_failed;
+        struct nfs3_fh                   fh       = {{0}, };
+        xlator_t                        *vol      = NULL;
+        nfs3_state_t                    *nfs3     = NULL;
+        nfs3_call_state_t               *cs       = NULL;
+        struct nfs_state                *nfs      = NULL;
 
         if (!req)
                 return ret;
@@ -1307,9 +1428,11 @@ nlm4svc_lock (rpcsvc_request_t *req)
                 rpcsvc_request_seterr (req, GARBAGE_ARGS);
                 goto rpcerr;
         }
+
         fh = cs->lockfh;
+        cs->monitor = mon;
         nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, &fh, req, vol, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
 
         if (nlm_grace_period && !cs->args.nlm4_lockargs.reclaim) {
                 gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
@@ -1341,12 +1464,26 @@ rpcerr:
         if (ret < 0) {
                 nfs3_call_state_wipe (cs);
         }
+
         return ret;
 }
 
 int
+nlm4svc_lock (rpcsvc_request_t *req)
+{
+        return nlm4svc_lock_common (req, 1);
+}
+
+int
+nlm4svc_nm_lock (rpcsvc_request_t *req)
+{
+        return nlm4svc_lock_common (req, 0);
+}
+
+int
 nlm4svc_cancel_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock)
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                    dict_t *xdata)
 {
         nlm4_stats                      stat = nlm4_denied;
         nfs3_call_state_t              *cs = NULL;
@@ -1368,7 +1505,6 @@ err:
 int
 nlm4_cancel_fd_resume (void *carg)
 {
-        nlm4_stats                      stat = nlm4_denied;
         int                             ret = -EFAULT;
         nfs_user_t                      nfu = {0, };
         nfs3_call_state_t               *cs = NULL;
@@ -1378,7 +1514,6 @@ nlm4_cancel_fd_resume (void *carg)
                 return ret;
 
         cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
         nfs_request_user_init (&nfu, cs->req);
         nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_cancargs.alock,
                                 cs->args.nlm4_cancargs.exclusive);
@@ -1387,22 +1522,112 @@ nlm4_cancel_fd_resume (void *carg)
         ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
                       &flock, nlm4svc_cancel_cbk, cs);
 
-        if (ret < 0)
+        return ret;
+}
+
+int
+nlm4_cancel_resume (void *carg)
+{
+        nlm4_stats                      stat = nlm4_failed;
+        int                             ret = -EFAULT;
+        nfs3_call_state_t               *cs = NULL;
+        nlm_client_t                    *nlmclnt = NULL;
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
+
+        nlmclnt = nlm_get_uniq (cs->args.nlm4_cancargs.alock.caller_name);
+        if (nlmclnt == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
+                goto nlm4err;
+        }
+        cs->fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
+        if (cs->fd == NULL) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "fd_lookup_uint64 retrned NULL");
+                goto nlm4err;
+        }
+        ret = nlm4_cancel_fd_resume (cs);
+
+nlm4err:
+        if (ret < 0) {
+                gf_log (GF_NLM, GF_LOG_WARNING, "unable to unlock_fd_resume()");
                 stat = nlm4_errno_to_nlm4stat (-ret);
+                nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
+                                    stat);
+
+                nfs3_call_state_wipe (cs);
+        }
+        /* clean up is taken care of */
+        return 0;
+}
+
+int
+nlm4svc_cancel (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        nlm4_stats                      stat = nlm4_failed;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        struct nfs3_fh                  fh = {{0}, };
+
+        if (!req)
+                return ret;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     stat, rpcerr);
+
+        nlm4_prep_nlm4_cancargs (&cs->args.nlm4_cancargs, &fh, &cs->lkowner,
+                                 cs->cookiebytes);
+        if (xdr_to_nlm4_cancelargs(req->msg[0], &cs->args.nlm4_cancargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+        if (nlm_grace_period) {
+                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
+                stat = nlm4_denied_grace_period;
+                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
+
+        cs->vol = vol;
+        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        ret = nfs3_fh_resolve_and_resume (cs, &fh,
+                                          NULL, nlm4_cancel_resume);
+
 nlm4err:
         if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to call lk()");
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
                 nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
                                     stat);
                 nfs3_call_state_wipe (cs);
+                return 0;
         }
 
+rpcerr:
+        if (ret < 0) {
+                nfs3_call_state_wipe (cs);
+        }
         return ret;
 }
 
 int
 nlm4svc_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock)
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                    dict_t *xdata)
 {
         nlm4_stats                      stat = nlm4_denied;
         nfs3_call_state_t              *cs = NULL;
@@ -1427,7 +1652,6 @@ err:
 int
 nlm4_unlock_fd_resume (void *carg)
 {
-        nlm4_stats                      stat = nlm4_denied;
         int                             ret = -EFAULT;
         nfs_user_t                      nfu = {0, };
         nfs3_call_state_t               *cs = NULL;
@@ -1436,7 +1660,6 @@ nlm4_unlock_fd_resume (void *carg)
         if (!carg)
                 return ret;
         cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
         nfs_request_user_init (&nfu, cs->req);
         nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_unlockargs.alock, 0);
         nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_unlockargs.alock.oh);
@@ -1444,21 +1667,11 @@ nlm4_unlock_fd_resume (void *carg)
         ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
                       &flock, nlm4svc_unlock_cbk, cs);
 
-        if (ret < 0)
-                stat = nlm4_errno_to_nlm4stat (-ret);
-nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to call lk()");
-                nlm4_generic_reply (cs->req, cs->args.nlm4_unlockargs.cookie,
-                                    stat);
-                nfs3_call_state_wipe (cs);
-        }
-
         return ret;
 }
 
 int
-nlm4_cancel_resume (void *carg)
+nlm4_unlock_resume (void *carg)
 {
         nlm4_stats                      stat = nlm4_failed;
         int                             ret = -1;
@@ -1471,31 +1684,36 @@ nlm4_cancel_resume (void *carg)
         cs = (nfs3_call_state_t *)carg;
         nlm4_check_fh_resolve_status (cs, stat, nlm4err);
 
-        nlmclnt = nlm_get_uniq (cs->args.nlm4_cancargs.alock.caller_name);
+        nlmclnt = nlm_get_uniq (cs->args.nlm4_unlockargs.alock.caller_name);
         if (nlmclnt == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
+                stat = nlm4_granted;
+                gf_log (GF_NLM, GF_LOG_WARNING, "nlm_get_uniq() returned NULL");
                 goto nlm4err;
         }
         cs->fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
         if (cs->fd == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
+                stat = nlm4_granted;
+                gf_log (GF_NLM, GF_LOG_WARNING, "fd_lookup_uint64() returned "
+                        "NULL");
                 goto nlm4err;
         }
-        ret = nlm4_cancel_fd_resume (cs);
+        ret = nlm4_unlock_fd_resume (cs);
 
 nlm4err:
         if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to unlock_fd_resume()");
-                nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
+                gf_log (GF_NLM, GF_LOG_WARNING, "unable to unlock_fd_resume");
+                stat = nlm4_errno_to_nlm4stat (-ret);
+                nlm4_generic_reply (cs->req, cs->args.nlm4_unlockargs.cookie,
                                     stat);
+
                 nfs3_call_state_wipe (cs);
         }
-
-        return ret;
+        /* we have already taken care of cleanup */
+        return 0;
 }
 
 int
-nlm4svc_cancel (rpcsvc_request_t *req)
+nlm4svc_unlock (rpcsvc_request_t *req)
 {
         xlator_t                        *vol = NULL;
         nlm4_stats                      stat = nlm4_failed;
@@ -1513,16 +1731,17 @@ nlm4svc_cancel (rpcsvc_request_t *req)
         nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
                                      stat, rpcerr);
 
-        nlm4_prep_nlm4_cancargs (&cs->args.nlm4_cancargs, &fh, &cs->lkowner,
-                                 cs->cookiebytes);
-        if (xdr_to_nlm4_cancelargs(req->msg[0], &cs->args.nlm4_cancargs) <= 0) {
+        nlm4_prep_nlm4_unlockargs (&cs->args.nlm4_unlockargs, &fh, &cs->lkowner,
+                                   cs->cookiebytes);
+        if (xdr_to_nlm4_unlockargs(req->msg[0], &cs->args.nlm4_unlockargs) <= 0)
+        {
                 gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
                 rpcsvc_request_seterr (req, GARBAGE_ARGS);
                 goto rpcerr;
         }
 
         nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, &fh, req, vol, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
 
         if (nlm_grace_period) {
                 gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
@@ -1533,16 +1752,17 @@ nlm4svc_cancel (rpcsvc_request_t *req)
         }
 
         cs->vol = vol;
+        /* FIXME: check if trans is being used at all for unlock */
+        cs->trans = rpcsvc_request_transport_ref(req);
         nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
 
         ret = nfs3_fh_resolve_and_resume (cs, &fh,
-                                          NULL, nlm4_cancel_resume);
+                                          NULL, nlm4_unlock_resume);
 
 nlm4err:
         if (ret < 0) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
-                nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
-                                    stat);
+                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
                 nfs3_call_state_wipe (cs);
                 return 0;
         }
@@ -1555,53 +1775,393 @@ rpcerr:
 }
 
 int
-nlm4_unlock_resume (void *carg)
+nlm4_share_reply (nfs3_call_state_t *cs, nlm4_stats stat)
 {
-        nlm4_stats                      stat = nlm4_failed;
-        int                             ret = -1;
-        nfs3_call_state_t               *cs = NULL;
-        nlm_client_t                    *nlmclnt = NULL;
+        nlm4_shareres  res = {{0}, 0, 0};
 
-        if (!carg)
+        if (!cs)
+                return -1;
+
+        res.cookie = cs->args.nlm4_shareargs.cookie;
+        res.stat = stat;
+        res.sequence = 0;
+
+        nlm4svc_submit_reply (cs->req, (void *)&res,
+                              (nlm4_serializer)xdr_serialize_nlm4_shareres);
+        return 0;
+}
+
+nlm_share_t *
+nlm4_share_new ()
+{
+        nlm_share_t      *share     = NULL;
+
+        share = GF_CALLOC (1, sizeof (nlm_share_t),
+                           gf_nfs_mt_nlm4_share);
+        if (!share)
+                goto out;
+
+        INIT_LIST_HEAD (&share->client_list);
+        INIT_LIST_HEAD (&share->inode_list);
+ out:
+        return share;
+}
+
+int
+nlm4_add_share_to_inode (nlm_share_t *share)
+{
+        int                    ret     = -1;
+        uint64_t               ctx     = 0;
+        struct list_head      *head    = NULL;
+        xlator_t              *this    = NULL;
+        inode_t               *inode   = NULL;
+        struct nfs_inode_ctx  *ictx    = NULL;
+        struct nfs_state      *priv    = NULL;
+
+        this = THIS;
+        priv = this->private;
+        inode = share->inode;
+        ret = inode_ctx_get (inode, this, &ctx);
+
+        if (ret == -1) {
+                ictx = GF_CALLOC (1, sizeof (struct nfs_inode_ctx),
+                                  gf_nfs_mt_inode_ctx);
+                if (!ictx ) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not allocate nfs inode ctx");
+                        ret = -1;
+                        goto out;
+                }
+                ictx->generation = priv->generation;
+
+                head = &ictx->shares;
+                INIT_LIST_HEAD (head);
+
+                ret = inode_ctx_put (inode, this, (uint64_t)ictx);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not store share list");
+                        goto out;
+                }
+        }
+        else {
+                ictx = (struct nfs_inode_ctx *)ctx;
+                head = &ictx->shares;
+        }
+
+        list_add (&share->inode_list, head);
+
+ out:
+        if (ret && head)
+                GF_FREE (head);
+
+        return ret;
+}
+
+int
+nlm4_approve_share_reservation (nfs3_call_state_t *cs)
+{
+        int                       ret               = -1;
+        uint64_t                  ctx               = 0;
+        fsh_mode                  req_mode          = 0;
+        fsh_access                req_access        = 0;
+        inode_t                  *inode             = NULL;
+        nlm_share_t              *share             = NULL;
+        struct list_head         *head              = NULL;
+        struct nfs_inode_ctx     *ictx              = NULL;
+
+        if (!cs)
+                goto out;
+
+        inode = cs->resolvedloc.inode;
+
+        ret = inode_ctx_get (inode, THIS, &ctx);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+        ictx = (struct nfs_inode_ctx *)ctx;
+
+        head = &ictx->shares;
+        if (!head || list_empty (head))
+                goto out;
+
+        req_mode = cs->args.nlm4_shareargs.share.mode;
+        req_access = cs->args.nlm4_shareargs.share.access;
+
+        list_for_each_entry (share, head, inode_list) {
+                ret = (((req_mode & share->access) == 0) &&
+                       ((req_access & share->mode) == 0));
+                if (!ret) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+
+ out:
+         return ret;
+}
+
+int
+nlm4_create_share_reservation (nfs3_call_state_t *cs)
+{
+        int                       ret        = -1;
+        nlm_share_t              *share      = NULL;
+        nlm_client_t             *client     = NULL;
+        inode_t                  *inode      = NULL;
+
+        LOCK (&nlm_client_list_lk);
+
+        inode = inode_ref (cs->resolvedloc.inode);
+        if (!inode) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "inode not found");
+                goto out;
+        }
+
+        client = __nlm_get_uniq (cs->args.nlm4_shareargs.share.caller_name);
+        if (!client) {
+                /* DO NOT add client. the client is supposed
+                   to be here, since nlm4svc_share adds it */
+                gf_log (GF_NLM, GF_LOG_ERROR, "client not found");
+                goto out;
+        }
+
+        ret = nlm4_approve_share_reservation (cs);
+        if (ret)
+                goto out;
+
+        share = nlm4_share_new ();
+        if (!share) {
+                ret = -1;
+                goto out;
+        }
+
+        share->inode  = inode;
+        share->mode   = cs->args.nlm4_shareargs.share.mode;
+        share->access = cs->args.nlm4_shareargs.share.access;
+        nlm_copy_lkowner (&share->lkowner,
+                          &cs->args.nlm4_shareargs.share.oh);
+
+        ret = nlm4_add_share_to_inode (share);
+        if (ret)
+                goto out;
+
+        list_add (&share->client_list, &client->shares);
+
+ out:
+        if (ret && inode) {
+                inode_unref (inode);
+                GF_FREE (share);
+        }
+
+        UNLOCK (&nlm_client_list_lk);
+        return ret;
+}
+
+/*
+  SHARE and UNSHARE calls DO NOT perform STACK_WIND,
+  the (non-monitored) share reservations are maintained
+  at *nfs xlator level only*, in memory
+*/
+int
+nlm4_share_resume (void *call_state)
+{
+        int                             ret             = -1;
+        nlm4_stats                      stat            = nlm4_failed;
+        nfs3_call_state_t              *cs              = NULL;
+
+        if (!call_state)
                 return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
+        cs = (nfs3_call_state_t *)call_state;
+        nlm4_check_fh_resolve_status (cs, stat, out);
 
-        nlmclnt = nlm_get_uniq (cs->args.nlm4_unlockargs.alock.caller_name);
-        if (nlmclnt == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
-                goto nlm4err;
+        ret = nlm4_create_share_reservation (cs);
+        if (!ret)
+                stat = nlm4_granted;
+
+ out:
+        nlm4_share_reply (cs, stat);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+nlm4svc_share (rpcsvc_request_t *req)
+{
+        nlm4_stats                    stat      = nlm4_failed;
+        xlator_t                     *vol       = NULL;
+        nfs3_state_t                 *nfs3      = NULL;
+        nfs3_call_state_t            *cs        = NULL;
+        struct nfs_state             *nfs       = NULL;
+        struct nfs3_fh                fh        = {{0}, };
+        int                           ret       = RPCSVC_ACTOR_ERROR;
+
+        if (!req)
+                return ret;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     stat, rpcerr);
+
+        nlm4_prep_shareargs (&cs->args.nlm4_shareargs, &cs->lockfh,
+                             &cs->lkowner, cs->cookiebytes);
+
+        if (xdr_to_nlm4_shareargs (req->msg[0],
+                                   &cs->args.nlm4_shareargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding SHARE args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
         }
-        cs->fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
-        if (cs->fd == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "fd_lookup_uint64() returned NULL");
-                goto nlm4err;
+
+        fh = cs->lockfh;
+        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req,
+                               vol, stat, nlm4err);
+
+        if (nlm_grace_period && !cs->args.nlm4_shareargs.reclaim) {
+                gf_log (GF_NLM, GF_LOG_DEBUG, "NLM in grace period");
+                stat = nlm4_denied_grace_period;
+                nlm4_share_reply (cs, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
         }
-        ret = nlm4_unlock_fd_resume (cs);
 
-nlm4err:
+        cs->vol = vol;
+        cs->trans = rpcsvc_request_transport_ref(req);
+        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        ret = nlm_add_nlmclnt (cs->args.nlm4_shareargs.share.caller_name);
+
+        ret = nfs3_fh_resolve_and_resume (cs, &fh, NULL, nlm4_share_resume);
+
+ nlm4err:
         if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to unlock_fd_resume");
-                nlm4_generic_reply (cs->req, cs->args.nlm4_unlockargs.cookie,
-                                    stat);
+                gf_log (GF_NLM, GF_LOG_ERROR, "SHARE call failed");
+                nlm4_share_reply (cs, stat);
+                nfs3_call_state_wipe (cs);
+                return 0;
+        }
 
+ rpcerr:
+        if (ret < 0)
                 nfs3_call_state_wipe (cs);
+
+        return ret;
+}
+
+int
+nlm4_remove_share_reservation (nfs3_call_state_t *cs)
+{
+        int                      ret        = -1;
+        uint64_t                 ctx        = 0;
+        fsh_mode                 req_mode   = 0;
+        fsh_access               req_access = 0;
+        nlm_share_t             *share      = NULL;
+        nlm_share_t             *tmp        = NULL;
+        nlm_client_t            *client     = NULL;
+        char                    *caller     = NULL;
+        inode_t                 *inode      = NULL;
+        xlator_t                *this       = NULL;
+        struct list_head        *head       = NULL;
+        nlm4_shareargs          *args       = NULL;
+        struct nfs_inode_ctx    *ictx       = NULL;
+
+        LOCK (&nlm_client_list_lk);
+
+        args = &cs->args.nlm4_shareargs;
+        caller = args->share.caller_name;
+
+        client = __nlm_get_uniq (caller);
+        if (!client) {
+                gf_log (GF_NLM, GF_LOG_ERROR,
+                        "client not found: %s", caller);
+                goto out;
+        }
+
+        inode = cs->resolvedloc.inode;
+        if (!inode) {
+                gf_log (GF_NLM, GF_LOG_ERROR,
+                        "inode not found: client: %s", caller);
+                goto out;
         }
 
+        this = THIS;
+        ret = inode_ctx_get (inode, this, &ctx);
+        if (ret) {
+                gf_log (GF_NLM, GF_LOG_ERROR,
+                        "no shares found for inode:"
+                        "gfid: %s; client: %s",
+                        inode->gfid, caller);
+                goto out;
+        }
+        ictx = (struct nfs_inode_ctx *)ctx;
+
+        head = &ictx->shares;
+        if (list_empty (head)) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+        req_mode = args->share.mode;
+        req_access = args->share.access;
+
+        list_for_each_entry_safe (share, tmp, head, inode_list) {
+                ret = ((req_mode == share->mode) &&
+                       (req_access == share->access) &&
+                       nlm_is_oh_same_lkowner (&share->lkowner, &args->share.oh));
+                if (ret) {
+                        list_del (&share->client_list);
+                        list_del (&share->inode_list);
+                        inode_unref (share->inode);
+                        GF_FREE (share);
+                        break;
+                }
+        }
+
+        ret = 0;
+ out:
+        UNLOCK (&nlm_client_list_lk);
         return ret;
+
 }
 
 int
-nlm4svc_unlock (rpcsvc_request_t *req)
+nlm4_unshare_resume (void *call_state)
 {
-        xlator_t                        *vol = NULL;
-        nlm4_stats                      stat = nlm4_failed;
-        struct nfs_state               *nfs = NULL;
-        nfs3_state_t                   *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-        int                             ret = RPCSVC_ACTOR_ERROR;
-        struct nfs3_fh                  fh = {{0}, };
+        int                ret        = -1;
+        nlm4_stats         stat       = nlm4_failed;
+        nfs3_call_state_t *cs         = NULL;
+
+        if (!call_state)
+                return ret;
+
+        cs = (nfs3_call_state_t *)call_state;
+
+        nlm4_check_fh_resolve_status (cs, stat, out);
+        ret = nlm4_remove_share_reservation (cs);
+        if (!ret)
+                stat = nlm4_granted;
+
+ out:
+        nlm4_share_reply (cs, stat);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+nlm4svc_unshare (rpcsvc_request_t *req)
+{
+        nlm4_stats                    stat      = nlm4_failed;
+        xlator_t                     *vol       = NULL;
+        nfs3_state_t                 *nfs3      = NULL;
+        nfs3_call_state_t            *cs        = NULL;
+        struct nfs_state             *nfs       = NULL;
+        struct nfs3_fh                fh        = {{0}, };
+        int                           ret       = RPCSVC_ACTOR_ERROR;
 
         if (!req)
                 return ret;
@@ -1611,22 +2171,25 @@ nlm4svc_unlock (rpcsvc_request_t *req)
         nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
                                      stat, rpcerr);
 
-        nlm4_prep_nlm4_unlockargs (&cs->args.nlm4_unlockargs, &fh, &cs->lkowner,
-                                   cs->cookiebytes);
-        if (xdr_to_nlm4_unlockargs(req->msg[0], &cs->args.nlm4_unlockargs) <= 0)
-        {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
+        nlm4_prep_shareargs (&cs->args.nlm4_shareargs, &cs->lockfh,
+                             &cs->lkowner, cs->cookiebytes);
+
+        if (xdr_to_nlm4_shareargs (req->msg[0],
+                                   &cs->args.nlm4_shareargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding UNSHARE args");
                 rpcsvc_request_seterr (req, GARBAGE_ARGS);
                 goto rpcerr;
         }
 
+        fh = cs->lockfh;
         nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, &fh, req, vol, stat, nlm4err);
+        nlm4_map_fh_to_volume (cs->nfs3state, fh, req,
+                               vol, stat, nlm4err);
 
-        if (nlm_grace_period) {
-                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
+        if (nlm_grace_period && !cs->args.nlm4_shareargs.reclaim) {
+                gf_log (GF_NLM, GF_LOG_DEBUG, "NLM in grace period");
                 stat = nlm4_denied_grace_period;
-                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
+                nlm4_share_reply (cs, stat);
                 nfs3_call_state_wipe (cs);
                 return 0;
         }
@@ -1635,22 +2198,90 @@ nlm4svc_unlock (rpcsvc_request_t *req)
         cs->trans = rpcsvc_request_transport_ref(req);
         nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &fh,
-                                          NULL, nlm4_unlock_resume);
+        ret = nfs3_fh_resolve_and_resume (cs, &fh, NULL,
+                                          nlm4_unshare_resume);
 
-nlm4err:
+ nlm4err:
         if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
-                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
-                nfs3_call_state_wipe (cs);
+                gf_log (GF_NLM, GF_LOG_ERROR, "UNSHARE call failed");
+                nlm4_share_reply (cs, stat);
+                ret = 0;
                 return 0;
         }
 
-rpcerr:
-        if (ret < 0) {
+ rpcerr:
+        if (ret < 0)
                 nfs3_call_state_wipe (cs);
+
+        return ret;
+}
+
+int
+nlm4_free_all_shares (char *caller_name)
+{
+        nlm_share_t             *share      = NULL;
+        nlm_share_t             *tmp        = NULL;
+        nlm_client_t            *client     = NULL;
+
+        LOCK (&nlm_client_list_lk);
+
+        client = __nlm_get_uniq (caller_name);
+        if (!client) {
+                gf_log (GF_NLM, GF_LOG_DEBUG,
+                        "client not found: %s", caller_name);
+                goto out;
+        }
+
+        list_for_each_entry_safe (share, tmp, &client->shares, client_list) {
+                list_del (&share->inode_list);
+                list_del (&share->client_list);
+                inode_unref (share->inode);
+                GF_FREE (share);
         }
+ out:
+        UNLOCK (&nlm_client_list_lk);
+        return 0;
+}
+
+int
+nlm4svc_free_all (rpcsvc_request_t *req)
+{
+        int                           ret       = RPCSVC_ACTOR_ERROR;
+        nlm4_stats                    stat      = nlm4_failed;
+        nfs3_state_t                 *nfs3      = NULL;
+        nfs3_call_state_t            *cs        = NULL;
+        struct nfs_state             *nfs       = NULL;
+
+        nlm4_validate_nfs3_state (req, nfs3, stat, err, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        nlm4_handle_call_state_init (nfs->nfs3state, cs,
+                                     req, stat, err);
+
+        nlm4_prep_freeallargs (&cs->args.nlm4_freeallargs,
+                               &cs->lkowner);
+
+        if (xdr_to_nlm4_freeallargs (req->msg[0],
+                                     &cs->args.nlm4_freeallargs) <= 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding FREE_ALL args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto err;
+        }
+
+        ret = nlm4_free_all_shares (cs->args.nlm4_freeallargs.name);
+        if (ret)
+                goto err;
+
+        ret = nlm_cleanup_fds (cs->args.nlm4_freeallargs.name);
+        if (ret)
+                goto err;
+
+ err:
+        nfs3_call_state_wipe (cs);
+        if (ret)
+                gf_log (GF_NLM, GF_LOG_DEBUG,
+                        "error in free all; stat: %d", stat);
         return ret;
+
 }
 
 void
@@ -1663,23 +2294,35 @@ nlm4svc_sm_notify (struct nlm_sm_status *status)
 }
 
 rpcsvc_actor_t  nlm4svc_actors[NLM4_PROC_COUNT] = {
-        {"NULL", NLM4_NULL, nlm4svc_null, NULL, NULL},
-        {"TEST", NLM4_TEST, nlm4svc_test, NULL, NULL},
-        {"LOCK", NLM4_LOCK, nlm4svc_lock, NULL, NULL},
-        {"CANCEL", NLM4_CANCEL, nlm4svc_cancel, NULL, NULL},
-        {"UNLOCK", NLM4_UNLOCK, nlm4svc_unlock, NULL, NULL},
-        {"GRANTED", NLM4_GRANTED, NULL, NULL, NULL},
-        {"TEST", NLM4_TEST_MSG, NULL, NULL, NULL},
-        {"LOCK", NLM4_LOCK_MSG, NULL, NULL, NULL},
-        {"CANCEL", NLM4_CANCEL_MSG, NULL, NULL, NULL},
-        {"UNLOCK", NLM4_UNLOCK_MSG, NULL, NULL, NULL},
-        {"GRANTED", NLM4_GRANTED_MSG, NULL, NULL, NULL},
-        {"TEST", NLM4_TEST_RES, NULL, NULL, NULL},
-        {"LOCK", NLM4_LOCK_RES, NULL, NULL, NULL},
-        {"CANCEL", NLM4_CANCEL_RES, NULL, NULL, NULL},
-        {"UNLOCK", NLM4_UNLOCK_RES, NULL, NULL, NULL},
-        {"GRANTED", NLM4_GRANTED_RES, NULL, NULL, NULL},
-        {"SM_NOTIFY", NLM4_SM_NOTIFY, NULL, NULL, NULL},
+        /* 0 */
+        {"NULL",       NLM4_NULL,         nlm4svc_null,      NULL, 0, DRC_IDEMPOTENT},
+        {"TEST",       NLM4_TEST,         nlm4svc_test,      NULL, 0, DRC_IDEMPOTENT},
+        {"LOCK",       NLM4_LOCK,         nlm4svc_lock,      NULL, 0, DRC_NON_IDEMPOTENT},
+        {"CANCEL",     NLM4_CANCEL,       nlm4svc_cancel,    NULL, 0, DRC_NON_IDEMPOTENT},
+        {"UNLOCK",     NLM4_UNLOCK,       nlm4svc_unlock,    NULL, 0, DRC_NON_IDEMPOTENT},
+        /* 5 */
+        {"GRANTED",    NLM4_GRANTED,      NULL,              NULL, 0, DRC_NA},
+        {"TEST",       NLM4_TEST_MSG,     NULL,              NULL, 0, DRC_NA},
+        {"LOCK",       NLM4_LOCK_MSG,     NULL,              NULL, 0, DRC_NA},
+        {"CANCEL",     NLM4_CANCEL_MSG,   NULL,              NULL, 0, DRC_NA},
+        {"UNLOCK",     NLM4_UNLOCK_MSG,   NULL,              NULL, 0, DRC_NA},
+        /* 10 */
+        {"GRANTED",    NLM4_GRANTED_MSG,  NULL,              NULL, 0, DRC_NA},
+        {"TEST",       NLM4_TEST_RES,     NULL,              NULL, 0, DRC_NA},
+        {"LOCK",       NLM4_LOCK_RES,     NULL,              NULL, 0, DRC_NA},
+        {"CANCEL",     NLM4_CANCEL_RES,   NULL,              NULL, 0, DRC_NA},
+        {"UNLOCK",     NLM4_UNLOCK_RES,   NULL,              NULL, 0, DRC_NA},
+        /* 15 ; procedures 17,18,19 are not defined by nlm */
+        {"GRANTED",    NLM4_GRANTED_RES,  NULL,              NULL, 0, DRC_NA},
+        {"SM_NOTIFY",  NLM4_SM_NOTIFY,    NULL,              NULL, 0, DRC_NA},
+        {"SEVENTEEN",  NLM4_SEVENTEEN,    NULL,              NULL, 0, DRC_NA},
+        {"EIGHTEEN",   NLM4_EIGHTEEN,     NULL,              NULL, 0, DRC_NA},
+        {"NINETEEN",   NLM4_NINETEEN,     NULL,              NULL, 0, DRC_NA},
+        /* 20 */
+        {"SHARE",      NLM4_SHARE,        nlm4svc_share,     NULL, 0, DRC_NON_IDEMPOTENT},
+        {"UNSHARE",    NLM4_UNSHARE,      nlm4svc_unshare,   NULL, 0, DRC_NON_IDEMPOTENT},
+        {"NM_LOCK",    NLM4_NM_LOCK,      nlm4svc_nm_lock,   NULL, 0, DRC_NON_IDEMPOTENT},
+        {"FREE_ALL",   NLM4_FREE_ALL,     nlm4svc_free_all,  NULL, 0, DRC_IDEMPOTENT},
 };
 
 rpcsvc_program_t        nlm4prog = {
@@ -1715,7 +2358,14 @@ nlm4svc_init(xlator_t *nfsx)
         int ret = -1;
         char *portstr = NULL;
         pthread_t thr;
-        struct timeval timeout = {0,};
+        struct timespec timeout = {0,};
+        FILE   *pidfile = NULL;
+        pid_t   pid     = -1;
+        static gf_boolean_t nlm4_inited = _gf_false;
+
+        /* Already inited */
+        if (nlm4_inited)
+                return &nlm4prog;
 
         nfs = (struct nfs_state*)nfsx->private;
 
@@ -1761,7 +2411,7 @@ nlm4svc_init(xlator_t *nfsx)
                 goto err;
         }
 
-        rpcsvc_create_listeners (nfs->rpcsvc, options, "NLM");
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, "NLM");
         if (ret == -1) {
                 gf_log (GF_NLM, GF_LOG_ERROR, "Unable to create listeners");
                 dict_unref (options);
@@ -1770,11 +2420,108 @@ nlm4svc_init(xlator_t *nfsx)
         INIT_LIST_HEAD(&nlm_client_list);
         LOCK_INIT (&nlm_client_list_lk);
 
+        /* unlink sm-notify.pid so that when we restart rpc.statd/sm-notify
+         * it thinks that the machine has restarted and sends NOTIFY to clients.
+         */
+        ret = unlink ("/var/run/sm-notify.pid");
+        if (ret == -1 && errno != ENOENT) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to unlink sm-notify");
+                goto err;
+        }
+        /* temporary work around to restart statd, not distro/OS independant.
+         * Need to figure out a more graceful way
+         * killall will cause problems on solaris.
+         */
+
+        pidfile = fopen ("/var/run/rpc.statd.pid", "r");
+        if (pidfile) {
+                ret = fscanf (pidfile, "%d", &pid);
+                if (ret <= 0) {
+                        gf_log (GF_NLM, GF_LOG_WARNING, "unable to get pid of "
+                                "rpc.statd");
+                        ret = runcmd ("killall", "-9", "rpc.statd", NULL);
+                } else
+                        kill (pid, SIGKILL);
+
+                fclose (pidfile);
+        } else {
+                gf_log (GF_NLM, GF_LOG_WARNING, "opening the pid file of "
+                        "rpc.statd failed (%s)", strerror (errno));
+                /* if ret == -1, do nothing - case either statd was not
+                 * running or was running in valgrind mode
+                 */
+                ret = runcmd ("killall", "-9", "rpc.statd", NULL);
+        }
+
+        ret = unlink ("/var/run/rpc.statd.pid");
+        if (ret == -1 && errno != ENOENT) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to unlink rpc.statd");
+                goto err;
+        }
+
+        ret = runcmd ("/sbin/rpc.statd", NULL);
+        if (ret == -1) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "unable to start rpc.statd");
+                goto err;
+        }
         pthread_create (&thr, NULL, nsm_thread, (void*)NULL);
 
         timeout.tv_sec = nlm_grace_period;
+        timeout.tv_nsec = 0;
+
         gf_timer_call_after (nfsx->ctx, timeout, nlm_grace_period_over, NULL);
+        nlm4_inited = _gf_true;
         return &nlm4prog;
 err:
         return NULL;
 }
+
+int32_t
+nlm_priv (xlator_t *this)
+{
+        int32_t                  ret                       = -1;
+        uint32_t                 client_count              = 0;
+        uint64_t                 file_count                = 0;
+        nlm_client_t            *client                    = NULL;
+        nlm_fde_t               *fde                       = NULL;
+        char                     key[GF_DUMP_MAX_BUF_LEN]  = {0};
+        char                     gfid_str[64]              = {0};
+
+        gf_proc_dump_add_section("nfs.nlm");
+
+        if (TRY_LOCK (&nlm_client_list_lk))
+                goto out;
+
+        list_for_each_entry (client, &nlm_client_list, nlm_clients) {
+
+                gf_proc_dump_build_key (key, "client", "%d.hostname", client_count);
+                gf_proc_dump_write (key, "%s\n", client->caller_name);
+
+                file_count = 0;
+                list_for_each_entry (fde, &client->fdes, fde_list) {
+                        gf_proc_dump_build_key (key, "file", "%ld.gfid", file_count);
+                        memset (gfid_str, 0, 64);
+                        uuid_utoa_r (fde->fd->inode->gfid, gfid_str);
+                        gf_proc_dump_write (key, "%s", gfid_str);
+                        file_count++;
+                }
+
+                gf_proc_dump_build_key (key, "client", "files-locked");
+                gf_proc_dump_write (key, "%ld\n", file_count);
+                client_count++;
+        }
+
+        gf_proc_dump_build_key (key, "nlm", "client-count");
+        gf_proc_dump_write (key, "%d", client_count);
+        ret = 0;
+        UNLOCK (&nlm_client_list_lk);
+
+ out:
+        if (ret) {
+                gf_proc_dump_build_key (key, "nlm", "statedump_error");
+                gf_proc_dump_write (key, "Unable to dump nlm state because "
+                                    "nlm_client_list_lk lock couldn't be acquired");
+        }
+
+        return ret;
+}
diff --git a/xlators/nfs/server/src/nlm4.h b/xlators/nfs/server/src/nlm4.h
index 3c00186d9..9b5d54081 100644
--- a/xlators/nfs/server/src/nlm4.h
+++ b/xlators/nfs/server/src/nlm4.h
@@ -2,19 +2,10 @@
   Copyright (c) 2012 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _NLM4_H_
@@ -25,6 +16,8 @@
 #include "config.h"
 #endif
 
+#include <sys/types.h>
+#include <signal.h>
 #include "rpcsvc.h"
 #include "dict.h"
 #include "xlator.h"
@@ -60,19 +53,25 @@ typedef struct nlm_client {
         pid_t uniq;
         struct list_head nlm_clients;
         struct list_head fdes;
+        struct list_head shares;
         struct rpc_clnt *rpc_clnt;
         char *caller_name;
+        int nsm_monitor;
 } nlm_client_t;
 
+typedef struct nlm_share {
+        struct list_head     client_list;
+        struct list_head     inode_list;
+        gf_lkowner_t         lkowner;
+        inode_t             *inode;
+        fsh_mode             mode;
+        fsh_access           access;
+} nlm_share_t;
+
 typedef struct nlm_fde {
         struct list_head fde_list;
         fd_t *fd;
         int transit_cnt;
 } nlm_fde_t;
 
-typedef struct {
-        pthread_cond_t cond;
-        pthread_mutex_t mutex;
-} nlm_condmutex_t;
-
 #endif
diff --git a/xlators/nfs/server/src/nlmcbk_svc.c b/xlators/nfs/server/src/nlmcbk_svc.c
index 5401dc39b..e1b588765 100644
--- a/xlators/nfs/server/src/nlmcbk_svc.c
+++ b/xlators/nfs/server/src/nlmcbk_svc.c
@@ -2,19 +2,10 @@
   Copyright (c) 2012 Gluster, Inc. <http://www.gluster.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /*
@@ -92,10 +83,14 @@ void *
 nsm_thread (void *argv)
 {
 	register SVCXPRT *transp;
+        int ret = 0;
 
-	pmap_unset (NLMCBK_PROGRAM, NLMCBK_V1);
-
-	transp = svcudp_create(RPC_ANYSOCK);
+        ret = pmap_unset (NLMCBK_PROGRAM, NLMCBK_V1);
+        if (ret == 0) {
+                gf_log (GF_NLM, GF_LOG_ERROR, "pmap_unset failed");
+                return NULL;
+        }
+        transp = svcudp_create(RPC_ANYSOCK);
 	if (transp == NULL) {
 		gf_log (GF_NLM, GF_LOG_ERROR, "cannot create udp service.");
                 return NULL;
diff --git a/xlators/performance/Makefile.am b/xlators/performance/Makefile.am
index eb94d8d6a..a494190ba 100644
--- a/xlators/performance/Makefile.am
+++ b/xlators/performance/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = write-behind read-ahead io-threads io-cache symlink-cache quick-read md-cache
+SUBDIRS = write-behind read-ahead readdir-ahead io-threads io-cache symlink-cache quick-read md-cache open-behind
 
 CLEANFILES = 
diff --git a/xlators/performance/io-cache/src/Makefile.am b/xlators/performance/io-cache/src/Makefile.am
index 6dd270e8f..155be9988 100644
--- a/xlators/performance/io-cache/src/Makefile.am
+++ b/xlators/performance/io-cache/src/Makefile.am
@@ -1,14 +1,16 @@
 xlator_LTLIBRARIES = io-cache.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-io_cache_la_LDFLAGS = -module -avoidversion 
+io_cache_la_LDFLAGS = -module -avoid-version 
 
 io_cache_la_SOURCES = io-cache.c page.c ioc-inode.c
 io_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = io-cache.h ioc-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/rbtree -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(CONTRIBDIR)/rbtree
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c
index 329668230..8febfc8fb 100644
--- a/xlators/performance/io-cache/src/io-cache.c
+++ b/xlators/performance/io-cache/src/io-cache.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -37,13 +28,10 @@ int ioc_log2_page_size;
 uint32_t
 ioc_get_priority (ioc_table_t *table, const char *path);
 
-uint32_t
-ioc_get_priority (ioc_table_t *table, const char *path);
-
 struct volume_options options[];
 
 
-inline uint32_t
+static inline uint32_t
 ioc_hashfn (void *data, int len)
 {
         off_t offset;
@@ -53,7 +41,7 @@ ioc_hashfn (void *data, int len)
         return (offset >> ioc_log2_page_size);
 }
 
-inline ioc_inode_t *
+static inline ioc_inode_t *
 ioc_inode_reupdate (ioc_inode_t *ioc_inode)
 {
         ioc_table_t *table = NULL;
@@ -66,7 +54,7 @@ ioc_inode_reupdate (ioc_inode_t *ioc_inode)
         return ioc_inode;
 }
 
-inline ioc_inode_t *
+static inline ioc_inode_t *
 ioc_get_inode (dict_t *dict, char *name)
 {
         ioc_inode_t *ioc_inode      = NULL;
@@ -157,15 +145,16 @@ ioc_inode_flush (ioc_inode_t *ioc_inode)
 int32_t
 ioc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *preop, struct iatt *postop)
+                 struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop,
+                             xdata);
         return 0;
 }
 
 int32_t
 ioc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid)
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         uint64_t ioc_inode = 0;
 
@@ -177,7 +166,7 @@ ioc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
 
         STACK_WIND (frame, ioc_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid);
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
 
         return 0;
 }
@@ -185,7 +174,7 @@ ioc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 int32_t
 ioc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret,	int32_t op_errno, inode_t *inode,
-                struct iatt *stbuf, dict_t *dict, struct iatt *postparent)
+                struct iatt *stbuf, dict_t *xdata, struct iatt *postparent)
 {
         ioc_inode_t *ioc_inode         = NULL;
         ioc_table_t *table             = NULL;
@@ -264,13 +253,13 @@ out:
         }
 
         STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, stbuf,
-                             dict, postparent);
+                             xdata, postparent);
         return 0;
 }
 
 int32_t
 ioc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            dict_t *xattr_req)
+            dict_t *xdata)
 {
         ioc_local_t *local    = NULL;
         int32_t      op_errno = -1, ret = -1;
@@ -292,7 +281,7 @@ ioc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         frame->local = local;
 
         STACK_WIND (frame, ioc_lookup_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lookup, loc, xattr_req);
+                    FIRST_CHILD (this)->fops->lookup, loc, xdata);
 
         return 0;
 
@@ -324,6 +313,20 @@ ioc_forget (xlator_t *this, inode_t *inode)
         return 0;
 }
 
+static int32_t
+ioc_invalidate(xlator_t *this, inode_t *inode)
+{
+	uint64_t     ioc_addr = 0;
+	ioc_inode_t *ioc_inode = NULL;
+
+	inode_ctx_get(inode, this, (uint64_t *) &ioc_addr);
+	ioc_inode = (void *) ioc_addr;
+
+	if (ioc_inode)
+		ioc_inode_flush(ioc_inode);
+
+	return 0;
+}
 
 /*
  * ioc_cache_validate_cbk -
@@ -338,7 +341,8 @@ ioc_forget (xlator_t *this, inode_t *inode)
  */
 int32_t
 ioc_cache_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
+                        int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                        dict_t *xdata)
 {
         ioc_local_t *local        = NULL;
         ioc_inode_t *ioc_inode    = NULL;
@@ -481,13 +485,13 @@ ioc_cache_validate (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
 
         STACK_WIND (validate_frame, ioc_cache_validate_cbk,
                     FIRST_CHILD (frame->this),
-                    FIRST_CHILD (frame->this)->fops->fstat, fd);
+                    FIRST_CHILD (frame->this)->fops->fstat, fd, NULL);
 
 out:
         return ret;
 }
 
-inline uint32_t
+static inline uint32_t
 is_match (const char *path, const char *pattern)
 {
         int32_t ret = 0;
@@ -528,7 +532,7 @@ ioc_get_priority (ioc_table_t *table, const char *path)
  */
 int32_t
 ioc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, fd_t *fd)
+              int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         uint64_t     tmp_ioc_inode = 0;
         ioc_local_t *local         = NULL;
@@ -549,6 +553,13 @@ ioc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
                 inode_ctx_get (fd->inode, this, &tmp_ioc_inode);
                 ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
 
+                //TODO: see why inode context is NULL and handle it.
+                if (!ioc_inode) {
+                        gf_log (this->name, GF_LOG_ERROR, "inode context is "
+                                "NULL (%s)", uuid_utoa (fd->inode->gfid));
+                        goto out;
+                }
+
                 ioc_table_lock (ioc_inode->table);
                 {
                         list_move_tail (&ioc_inode->inode_lru,
@@ -573,10 +584,6 @@ ioc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
                          */
                         fd_ctx_set (fd, this, 1);
                 }
-                if ((local->wbflags & GF_OPEN_NOWB) != 0) {
-                        /* disable caching as asked by NFS */
-                        fd_ctx_set (fd, this, 1);
-                }
 
                 /* weight = 0, we disable caching on it */
                 if (weight == 0) {
@@ -590,7 +597,7 @@ out:
         mem_put (local);
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 
         return 0;
 }
@@ -612,7 +619,7 @@ int32_t
 ioc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret,	int32_t op_errno, fd_t *fd,
                 inode_t *inode,	struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
         ioc_local_t *local     = NULL;
         ioc_table_t *table     = NULL;
@@ -687,7 +694,7 @@ out:
         mem_put (local);
 
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         return 0;
 }
@@ -697,7 +704,7 @@ int32_t
 ioc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, inode_t *inode,
                struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         ioc_local_t *local     = NULL;
         ioc_table_t *table     = NULL;
@@ -740,14 +747,14 @@ out:
         mem_put (local);
 
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 ioc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t rdev, dict_t *params)
+           dev_t rdev, mode_t umask, dict_t *xdata)
 {
         ioc_local_t *local    = NULL;
         int32_t      op_errno = -1, ret = -1;
@@ -771,7 +778,7 @@ ioc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         STACK_WIND (frame, ioc_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, params);
+                    loc, mode, rdev, umask, xdata);
         return 0;
 
 unwind:
@@ -781,7 +788,7 @@ unwind:
         }
 
         STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                             NULL, NULL);
+                             NULL, NULL, NULL);
 
         return 0;
 }
@@ -797,7 +804,7 @@ unwind:
  */
 int32_t
 ioc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+          fd_t *fd, dict_t *xdata)
 {
 
         ioc_local_t *local = NULL;
@@ -805,19 +812,19 @@ ioc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         local = mem_get0 (this->local_pool);
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
         }
 
         local->flags = flags;
         local->file_loc.path = loc->path;
         local->file_loc.inode = loc->inode;
-        local->wbflags = wbflags;
 
         frame->local = local;
 
         STACK_WIND (frame, ioc_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd,
+                    xdata);
 
         return 0;
 }
@@ -834,7 +841,7 @@ ioc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
  */
 int32_t
 ioc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-            mode_t mode, fd_t *fd, dict_t *params)
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         ioc_local_t *local = NULL;
 
@@ -842,7 +849,7 @@ ioc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
                 STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL,
-                                     NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -852,7 +859,7 @@ ioc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
 
         STACK_WIND (frame, ioc_create_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create, loc, flags, mode,
-                    fd, params);
+                    umask, fd, xdata);
 
         return 0;
 }
@@ -889,10 +896,10 @@ int32_t
 ioc_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret,	int32_t op_errno, struct iovec *vector,
                         int32_t count, struct iatt *stbuf,
-                        struct iobref *iobref)
+                        struct iobref *iobref, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+                             stbuf, iobref, xdata);
         return 0;
 }
 
@@ -984,6 +991,7 @@ ioc_dispatch_requests (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
                                                 "out of memory");
                                         local->op_ret = -1;
                                         local->op_errno = ENOMEM;
+                                        ioc_inode_unlock (ioc_inode);
                                         goto out;
                                 }
                         }
@@ -999,7 +1007,8 @@ ioc_dispatch_requests (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
                                                 "cache hit for trav_offset=%"
                                                 PRId64"/local_offset=%"PRId64"",
                                                 trav_offset, local_offset);
-                                        waitq = __ioc_page_wakeup (trav, 0);
+                                        waitq = __ioc_page_wakeup (trav,
+                                                                   trav->op_errno);
                                 } else {
                                         /* if waitq already exists, fstat
                                          * revalidate is
@@ -1017,7 +1026,7 @@ ioc_dispatch_requests (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
                                                 need_validate = 0;
 
                                                 waitq = __ioc_page_wakeup (trav,
-                                                                           0);
+                                                                           trav->op_errno);
                                                 ioc_inode_unlock (ioc_inode);
 
                                                 ioc_waitq_return (waitq);
@@ -1049,7 +1058,8 @@ ioc_dispatch_requests (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
                         if (ret == -1) {
                                 ioc_inode_lock (ioc_inode);
                                 {
-                                        waitq = __ioc_page_wakeup (trav, 0);
+                                        waitq = __ioc_page_wakeup (trav,
+                                                                   trav->op_errno);
                                 }
                                 ioc_inode_unlock (ioc_inode);
 
@@ -1085,7 +1095,7 @@ out:
  */
 int32_t
 ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           size_t size, off_t offset, uint32_t flags)
+           size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         uint64_t     tmp_ioc_inode = 0;
         ioc_inode_t *ioc_inode     = NULL;
@@ -1105,7 +1115,7 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 STACK_WIND (frame, ioc_readv_disabled_cbk,
                             FIRST_CHILD (frame->this),
                             FIRST_CHILD (frame->this)->fops->readv, fd, size,
-                            offset, flags);
+                            offset, flags, xdata);
                 return 0;
         }
 
@@ -1141,7 +1151,7 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 STACK_WIND (frame, ioc_readv_disabled_cbk,
                             FIRST_CHILD (frame->this),
                             FIRST_CHILD (frame->this)->fops->readv, fd, size,
-                            offset, flags);
+                            offset, flags, xdata);
                 return 0;
         }
 
@@ -1178,7 +1188,8 @@ ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 
 out:
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                             NULL);
         return 0;
 }
 
@@ -1195,7 +1206,7 @@ out:
 int32_t
 ioc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret,	int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+                struct iatt *postbuf, dict_t *xdata)
 {
         ioc_local_t *local     = NULL;
         uint64_t     ioc_inode = 0;
@@ -1206,7 +1217,8 @@ ioc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (ioc_inode)
                 ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
         return 0;
 }
 
@@ -1224,7 +1236,7 @@ ioc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref)
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         ioc_local_t *local     = NULL;
         uint64_t     ioc_inode = 0;
@@ -1233,7 +1245,7 @@ ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (local == NULL) {
                 gf_log (this->name, GF_LOG_ERROR, "out of memory");
 
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
                 return 0;
         }
 
@@ -1247,7 +1259,7 @@ ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         STACK_WIND (frame, ioc_writev_cbk, FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
-                    flags, iobref);
+                    flags, iobref, xdata);
 
         return 0;
 }
@@ -1266,11 +1278,11 @@ ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int32_t
 ioc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
 
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
         return 0;
 }
 
@@ -1289,11 +1301,11 @@ ioc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 ioc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
 
         STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
         return 0;
 }
 
@@ -1308,7 +1320,8 @@ ioc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  *
  */
 int32_t
-ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
         uint64_t ioc_inode = 0;
 
@@ -1318,7 +1331,7 @@ ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
                 ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
 
         STACK_WIND (frame, ioc_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
         return 0;
 }
 
@@ -1332,7 +1345,8 @@ ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
  *
  */
 int32_t
-ioc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+ioc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
 {
         uint64_t ioc_inode = 0;
 
@@ -1342,21 +1356,21 @@ ioc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
 
         STACK_WIND (frame, ioc_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
         return 0;
 }
 
 int32_t
 ioc_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-            int32_t op_errno, struct gf_flock *lock)
+            int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
 int32_t
 ioc_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-        struct gf_flock *lock)
+        struct gf_flock *lock, dict_t *xdata)
 {
         ioc_inode_t *ioc_inode = NULL;
         uint64_t     tmp_inode = 0;
@@ -1366,7 +1380,7 @@ ioc_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         if (!ioc_inode) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "inode context is NULL: returning EBADFD");
-                STACK_UNWIND_STRICT (lk, frame, -1, EBADFD, NULL);
+                STACK_UNWIND_STRICT (lk, frame, -1, EBADFD, NULL, NULL);
                 return 0;
         }
 
@@ -1377,14 +1391,14 @@ ioc_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         ioc_inode_unlock (ioc_inode);
 
         STACK_WIND (frame, ioc_lk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lk, fd, cmd, lock);
+                    FIRST_CHILD (this)->fops->lk, fd, cmd, lock, xdata);
 
         return 0;
 }
 
 int
 ioc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, gf_dirent_t *entries)
+                  int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         gf_dirent_t *entry = NULL;
 
@@ -1396,7 +1410,7 @@ ioc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 
         return 0;
 }
@@ -1411,6 +1425,58 @@ ioc_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         return 0;
 }
 
+static int32_t
+ioc_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		int32_t op_ret, int32_t op_errno, struct iatt *pre,
+		struct iatt *post, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, pre, post, xdata);
+	return 0;
+}
+
+static int32_t
+ioc_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
+{
+	uint64_t ioc_inode = 0;
+
+	inode_ctx_get (fd->inode, this, &ioc_inode);
+
+	if (ioc_inode)
+		ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+
+	STACK_WIND(frame, ioc_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+       return 0;
+}
+
+static int32_t
+ioc_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                struct iatt *post, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret,
+                            op_errno, pre, post, xdata);
+        return 0;
+}
+
+static int32_t
+ioc_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+        uint64_t ioc_inode = 0;
+
+        inode_ctx_get (fd->inode, this, &ioc_inode);
+
+        if (ioc_inode)
+                ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+
+        STACK_WIND(frame, ioc_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+       return 0;
+}
+
+
 int32_t
 ioc_get_priority_list (const char *opt_str, struct list_head *first)
 {
@@ -1491,13 +1557,9 @@ ioc_get_priority_list (const char *opt_str, struct list_head *first)
                 stripe_str = strtok_r (NULL, ",", &tmp_str);
         }
 out:
-        if (string != NULL) {
-                GF_FREE (string);
-        }
+        GF_FREE (string);
 
-        if (dup_str != NULL) {
-                GF_FREE (dup_str);
-        }
+        GF_FREE (dup_str);
 
         if (max_pri == -1) {
                 list_for_each_entry_safe (curr, tmp, first, list) {
@@ -1530,7 +1592,7 @@ mem_acct_init (xlator_t *this)
 }
 
 
-gf_boolean_t
+static gf_boolean_t
 check_cache_size_ok (xlator_t *this, uint64_t cache_size)
 {
         gf_boolean_t            ret = _gf_true;
@@ -1553,7 +1615,7 @@ check_cache_size_ok (xlator_t *this, uint64_t cache_size)
         else
                 max_cache_size = total_mem;
 
-        gf_log (this->name, GF_LOG_INFO, "Max cache size is %"PRIu64,
+        gf_log (this->name, GF_LOG_DEBUG, "Max cache size is %"PRIu64,
                 max_cache_size);
 
         if (cache_size > max_cache_size) {
@@ -1806,7 +1868,16 @@ void
 __ioc_page_dump (ioc_page_t *page, char *prefix)
 {
 
-        ioc_page_lock (page);
+        int    ret = -1;
+
+        if (!page)
+                return;
+        /* ioc_page_lock can be used to hold the mutex. But in statedump
+         * its better to use trylock to avoid deadlocks.
+         */
+        ret = pthread_mutex_trylock (&page->page_lock);
+        if (ret)
+                goto out;
         {
                 gf_proc_dump_write ("offset", "%"PRId64, page->offset);
                 gf_proc_dump_write ("size", "%"PRId64, page->size);
@@ -1814,7 +1885,14 @@ __ioc_page_dump (ioc_page_t *page, char *prefix)
                 gf_proc_dump_write ("ready", "%s", page->ready ? "yes" : "no");
                 ioc_page_waitq_dump (page, prefix);
         }
-        ioc_page_unlock (page);
+        pthread_mutex_unlock (&page->page_lock);
+
+out:
+        if (ret && page)
+                gf_proc_dump_write ("Unable to dump the page information",
+                                    "(Lock acquisition failed) %p", page);
+
+        return;
 }
 
 void
@@ -1824,7 +1902,6 @@ __ioc_cache_dump (ioc_inode_t *ioc_inode, char *prefix)
         ioc_table_t *table                    = NULL;
         ioc_page_t  *page                     = NULL;
         int          i                        = 0;
-        struct tm   *tm                       = NULL;
         char         key[GF_DUMP_MAX_BUF_LEN] = {0, };
         char         timestr[256]             = {0, };
 
@@ -1834,12 +1911,15 @@ __ioc_cache_dump (ioc_inode_t *ioc_inode, char *prefix)
 
         table = ioc_inode->table;
 
-        tm = localtime (&ioc_inode->cache.tv.tv_sec);
-        strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
-        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                  ".%"GF_PRI_SUSECONDS, ioc_inode->cache.tv.tv_usec);
+        if (ioc_inode->cache.tv.tv_sec) {
+                gf_time_fmt (timestr, sizeof timestr,
+                             ioc_inode->cache.tv.tv_sec, gf_timefmt_FT);
+                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, ioc_inode->cache.tv.tv_usec);
 
-        gf_proc_dump_write ("last-cache-validation-time", "%s", timestr);
+                gf_proc_dump_write ("last-cache-validation-time", "%s",
+                                    timestr);
+        }
 
         for (offset = 0; offset < ioc_inode->ia_size;
              offset += table->page_size) {
@@ -1856,41 +1936,91 @@ out:
 }
 
 
-void
-ioc_inode_dump (ioc_inode_t *ioc_inode, char *prefix)
+int
+ioc_inode_dump (xlator_t *this, inode_t *inode)
 {
 
-       if ((ioc_inode == NULL) || (prefix == NULL)) {
+        char         *path                            = NULL;
+        int           ret                             = -1;
+        char          key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+        uint64_t      tmp_ioc_inode                   = 0;
+        ioc_inode_t  *ioc_inode                       = NULL;
+        gf_boolean_t  section_added                   = _gf_false;
+        char          uuid_str[64]                    = {0,};
+
+        if (this == NULL || inode == NULL)
+                goto out;
+
+        gf_proc_dump_build_key (key_prefix, "io-cache", "inode");
+
+        inode_ctx_get (inode, this, &tmp_ioc_inode);
+        ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
+        if (ioc_inode == NULL)
+                goto out;
+
+        /* Similar to ioc_page_dump function its better to use
+         * pthread_mutex_trylock and not to use gf_log in statedump
+         * to avoid deadlocks.
+         */
+        ret = pthread_mutex_trylock (&ioc_inode->inode_lock);
+        if (ret)
                 goto out;
-        }
 
-        ioc_inode_lock (ioc_inode);
         {
+                if (uuid_is_null (ioc_inode->inode->gfid))
+                        goto unlock;
+
+                gf_proc_dump_add_section (key_prefix);
+                section_added = _gf_true;
+
+                __inode_path (ioc_inode->inode, NULL, &path);
+
                 gf_proc_dump_write ("inode.weight", "%d", ioc_inode->weight);
-                __ioc_cache_dump (ioc_inode, prefix);
-                __ioc_inode_waitq_dump (ioc_inode, prefix);
+
+                if (path) {
+                        gf_proc_dump_write ("path", "%s", path);
+                        GF_FREE (path);
+                }
+
+                gf_proc_dump_write ("uuid", "%s", uuid_utoa_r
+                                    (ioc_inode->inode->gfid, uuid_str));
+                __ioc_cache_dump (ioc_inode, key_prefix);
+                __ioc_inode_waitq_dump (ioc_inode, key_prefix);
         }
-        ioc_inode_unlock (ioc_inode);
+unlock:
+        pthread_mutex_unlock (&ioc_inode->inode_lock);
+
 out:
-        return;
+        if (ret && ioc_inode) {
+                if (section_added == _gf_false)
+                        gf_proc_dump_add_section (key_prefix);
+                gf_proc_dump_write ("Unable to print the status of ioc_inode",
+                                    "(Lock acquisition failed) %s",
+                                    uuid_utoa (inode->gfid));
+        }
+        return ret;
 }
 
 int
 ioc_priv_dump (xlator_t *this)
 {
         ioc_table_t *priv                            = NULL;
-        ioc_inode_t *ioc_inode                       = NULL;
         char         key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+        int          ret                             = -1;
+        gf_boolean_t add_section                     = _gf_false;
 
         if (!this || !this->private)
                 goto out;
 
         priv = this->private;
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.io-cache",
-                                "priv");
+
+        gf_proc_dump_build_key (key_prefix, "io-cache", "priv");
         gf_proc_dump_add_section (key_prefix);
+        add_section = _gf_true;
 
-        ioc_table_lock (priv);
+        ret = pthread_mutex_trylock (&priv->table_lock);
+        if (ret)
+                goto out;
         {
                 gf_proc_dump_write ("page_size", "%ld", priv->page_size);
                 gf_proc_dump_write ("cache_size", "%ld", priv->cache_size);
@@ -1899,13 +2029,20 @@ ioc_priv_dump (xlator_t *this)
                 gf_proc_dump_write ("cache_timeout", "%u", priv->cache_timeout);
                 gf_proc_dump_write ("min-file-size", "%u", priv->min_file_size);
                 gf_proc_dump_write ("max-file-size", "%u", priv->max_file_size);
-
-                list_for_each_entry (ioc_inode, &priv->inodes, inode_list) {
-                        ioc_inode_dump (ioc_inode, key_prefix);
-                }
         }
-        ioc_table_unlock (priv);
+        pthread_mutex_unlock (&priv->table_lock);
 out:
+        if (ret && priv) {
+                if (!add_section) {
+                        gf_proc_dump_build_key (key_prefix, "xlator."
+                                                "performance.io-cache", "priv");
+                        gf_proc_dump_add_section (key_prefix);
+                }
+                gf_proc_dump_write ("Unable to dump the state of private "
+                                    "structure of io-cache xlator", "(Lock "
+                                    "acquisition failed) %s", this->name);
+        }
+
         return 0;
 }
 
@@ -1918,18 +2055,33 @@ out:
 void
 fini (xlator_t *this)
 {
-        ioc_table_t *table = NULL;
+        ioc_table_t         *table = NULL;
+        struct ioc_priority *curr  = NULL, *tmp = NULL;
+        int                  i     = 0;
 
         table = this->private;
 
         if (table == NULL)
                 return;
 
+        this->private = NULL;
+
         if (table->mem_pool != NULL) {
                 mem_pool_destroy (table->mem_pool);
                 table->mem_pool = NULL;
         }
 
+        list_for_each_entry_safe (curr, tmp, &table->priority_list, list) {
+                list_del_init (&curr->list);
+                GF_FREE (curr->pattern);
+                GF_FREE (curr);
+        }
+
+        for (i = 0; i < table->max_pri; i++) {
+                GF_ASSERT (list_empty (&table->inode_lru[i]));
+        }
+
+        GF_ASSERT (list_empty (&table->inodes));
         pthread_mutex_destroy (&table->table_lock);
         GF_FREE (table);
 
@@ -1950,21 +2102,25 @@ struct xlator_fops fops = {
         .mknod       = ioc_mknod,
 
         .readdirp    = ioc_readdirp,
+	.discard     = ioc_discard,
+        .zerofill    = ioc_zerofill,
 };
 
 
 struct xlator_dumpops dumpops = {
         .priv        = ioc_priv_dump,
+        .inodectx    = ioc_inode_dump,
 };
 
 struct xlator_cbks cbks = {
         .forget      = ioc_forget,
-        .release     = ioc_release
+        .release     = ioc_release,
+	.invalidate  = ioc_invalidate,
 };
 
 struct volume_options options[] = {
         { .key  = {"priority"},
-          .type = GF_OPTION_TYPE_ANY,
+          .type = GF_OPTION_TYPE_PRIORITY_LIST,
           .default_value = "",
           .description = "Assigns priority to filenames with specific "
           "patterns so that when a page needs to be ejected "
diff --git a/xlators/performance/io-cache/src/io-cache.h b/xlators/performance/io-cache/src/io-cache.h
index bd60aa4f4..46d758a66 100644
--- a/xlators/performance/io-cache/src/io-cache.h
+++ b/xlators/performance/io-cache/src/io-cache.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __IO_CACHE_H
@@ -84,7 +75,6 @@ struct ioc_fill {
 struct ioc_local {
         mode_t           mode;
         int32_t          flags;
-        int32_t          wbflags;
         loc_t            file_loc;
         off_t            offset;
         size_t           size;
@@ -126,6 +116,8 @@ struct ioc_page {
         struct ioc_waitq    *waitq;
         struct iobref       *iobref;
         pthread_mutex_t     page_lock;
+        int32_t             op_errno;
+        char                stale;
 };
 
 struct ioc_cache {
@@ -196,7 +188,7 @@ int32_t
 ioc_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iovec *vector,
                         int32_t count, struct iatt *stbuf,
-                        struct iobref *iobref);
+                        struct iobref *iobref, dict_t *xdata);
 
 ioc_page_t *
 __ioc_page_get (ioc_inode_t *ioc_inode, off_t offset);
@@ -337,6 +329,4 @@ ioc_prune (ioc_table_t *table);
 int32_t
 ioc_need_prune (ioc_table_t *table);
 
-inline uint32_t
-ioc_hashfn (void *data, int len);
 #endif /* __IO_CACHE_H */
diff --git a/xlators/performance/io-cache/src/ioc-inode.c b/xlators/performance/io-cache/src/ioc-inode.c
index a99d5ea99..86a54bb14 100644
--- a/xlators/performance/io-cache/src/ioc-inode.c
+++ b/xlators/performance/io-cache/src/ioc-inode.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -124,7 +115,7 @@ ioc_inode_wakeup (call_frame_t *frame, ioc_inode_t *ioc_inode,
                                 {
                                         page_waitq =
                                                 __ioc_page_wakeup (waiter_page,
-                                                                   0);
+                                                                   waiter_page->op_errno);
                                 }
                                 ioc_inode_unlock (ioc_inode);
                                 if (page_waitq)
diff --git a/xlators/performance/io-cache/src/ioc-mem-types.h b/xlators/performance/io-cache/src/ioc-mem-types.h
index 0cae40b5c..9b68f9fce 100644
--- a/xlators/performance/io-cache/src/ioc-mem-types.h
+++ b/xlators/performance/io-cache/src/ioc-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __IOC_MT_H__
diff --git a/xlators/performance/io-cache/src/page.c b/xlators/performance/io-cache/src/page.c
index d3d74653a..416cd5fe4 100644
--- a/xlators/performance/io-cache/src/page.c
+++ b/xlators/performance/io-cache/src/page.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -111,6 +102,7 @@ __ioc_page_destroy (ioc_page_t *page)
         if (page->waitq) {
                 /* frames waiting on this page, do not destroy this page */
                 page_size = -1;
+                page->stale = 1;
         } else {
                 rbthash_remove (page->inode->cache.page_table, &page->offset,
                                 sizeof (page->offset));
@@ -144,6 +136,7 @@ int64_t
 ioc_page_destroy (ioc_page_t *page)
 {
         int64_t ret = 0;
+        struct ioc_inode *inode = NULL;
 
         if (page == NULL) {
                 goto out;
@@ -151,9 +144,10 @@ ioc_page_destroy (ioc_page_t *page)
 
         ioc_inode_lock (page->inode);
         {
+                inode = page->inode;
                 ret = __ioc_page_destroy (page);
         }
-        ioc_inode_unlock (page->inode);
+        ioc_inode_unlock (inode);
 
 out:
         return ret;
@@ -323,6 +317,7 @@ __ioc_wait_on_page (ioc_page_t *page, call_frame_t *frame, off_t offset,
                 local->op_errno = ENOMEM;
                 gf_log (frame->this->name, GF_LOG_WARNING,
                         "asked to wait on a NULL page");
+                goto out;
         }
 
         waitq = GF_CALLOC (1, sizeof (*waitq), gf_ioc_mt_ioc_waitq_t);
@@ -416,7 +411,8 @@ ioc_waitq_return (ioc_waitq_t *waitq)
 int
 ioc_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iovec *vector,
-               int32_t count, struct iatt *stbuf, struct iobref *iobref)
+               int32_t count, struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
 {
         ioc_local_t *local            = NULL;
         off_t        offset           = 0;
@@ -483,6 +479,7 @@ ioc_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         iobref_unref (page->iobref);
                                         GF_FREE (page->vector);
                                         page->vector = NULL;
+                                        page->iobref = NULL;
                                 }
 
                                 /* keep a copy of the page for our cache */
@@ -517,6 +514,7 @@ ioc_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                  * byte replies */
                                 page_size = iov_length(vector, count);
                                 page->size = page_size;
+                                page->op_errno = op_errno;
 
                                 iobref_page_size = iobref_size (page->iobref);
 
@@ -630,7 +628,7 @@ ioc_page_fault (ioc_inode_t *ioc_inode, call_frame_t *frame, fd_t *fd,
 
         STACK_WIND (fault_frame, ioc_fault_cbk, FIRST_CHILD(fault_frame->this),
                     FIRST_CHILD(fault_frame->this)->fops->readv, fd,
-                    table->page_size, offset, 0);
+                    table->page_size, offset, 0, NULL);
         return;
 
 err:
@@ -810,7 +808,7 @@ ioc_frame_unwind (call_frame_t *frame)
         int32_t        copied = 0;
         struct iobref *iobref = NULL;
         struct iatt    stbuf  = {0,};
-        int32_t        op_ret = 0;
+        int32_t        op_ret = 0, op_errno = 0;
 
         GF_ASSERT (frame);
 
@@ -819,16 +817,21 @@ ioc_frame_unwind (call_frame_t *frame)
                 gf_log (frame->this->name, GF_LOG_WARNING,
                         "local is NULL");
                 op_ret = -1;
-                local->op_errno = ENOMEM;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        if (local->op_ret < 0) {
+                op_ret = local->op_ret;
+                op_errno = local->op_errno;
                 goto unwind;
         }
 
         //  ioc_local_lock (local);
-        frame->local = NULL;
         iobref = iobref_new ();
         if (iobref == NULL) {
                 op_ret = -1;
-                local->op_errno = ENOMEM;
+                op_errno = ENOMEM;
         }
 
         if (list_empty (&local->fill_list)) {
@@ -845,7 +848,7 @@ ioc_frame_unwind (call_frame_t *frame)
         vector = GF_CALLOC (count, sizeof (*vector), gf_ioc_mt_iovec);
         if (vector == NULL) {
                 op_ret = -1;
-                local->op_errno = ENOMEM;
+                op_errno = ENOMEM;
         }
 
         list_for_each_entry_safe (fill, next, &local->fill_list, list) {
@@ -856,7 +859,10 @@ ioc_frame_unwind (call_frame_t *frame)
 
                         copied += (fill->count * sizeof (*vector));
 
-                        iobref_merge (iobref, fill->iobref);
+                        if (iobref_merge (iobref, fill->iobref)) {
+				op_ret = -1;
+				op_errno = ENOMEM;
+			}
                 }
 
                 list_del (&fill->list);
@@ -875,8 +881,9 @@ unwind:
 
         //  ioc_local_unlock (local);
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, local->op_errno, vector,
-                             count, &stbuf, iobref);
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, &stbuf, iobref, NULL);
 
         if (iobref != NULL) {
                 iobref_unref (iobref);
@@ -888,7 +895,8 @@ unwind:
         }
 
         pthread_mutex_destroy (&local->local_lock);
-        mem_put (local);
+        if (local)
+                mem_put (local);
 
         return;
 }
@@ -955,6 +963,10 @@ __ioc_page_wakeup (ioc_page_t *page, int32_t op_errno)
                 }
         }
 
+        if (page->stale) {
+                __ioc_page_destroy (page);
+        }
+
 out:
         return waitq;
 }
@@ -982,7 +994,7 @@ __ioc_page_error (ioc_page_t *page, int32_t op_ret, int32_t op_errno)
         waitq = page->waitq;
         page->waitq = NULL;
 
-        gf_log (page->inode->table->xl->name, GF_LOG_WARNING,
+        gf_log (page->inode->table->xl->name, GF_LOG_DEBUG,
                 "page error for page = %p & waitq = %p", page, waitq);
 
         for (trav = waitq; trav; trav = trav->next) {
@@ -1022,6 +1034,7 @@ ioc_waitq_t *
 ioc_page_error (ioc_page_t *page, int32_t op_ret, int32_t op_errno)
 {
         ioc_waitq_t  *waitq = NULL;
+        struct ioc_inode *inode = NULL;
 
         if (page == NULL) {
                 goto out;
@@ -1029,9 +1042,10 @@ ioc_page_error (ioc_page_t *page, int32_t op_ret, int32_t op_errno)
 
         ioc_inode_lock (page->inode);
         {
+                inode = page->inode;
                 waitq = __ioc_page_error (page, op_ret, op_errno);
         }
-        ioc_inode_unlock (page->inode);
+        ioc_inode_unlock (inode);
 
 out:
         return waitq;
diff --git a/xlators/performance/io-threads/src/Makefile.am b/xlators/performance/io-threads/src/Makefile.am
index 72f9a8012..d63042e7c 100644
--- a/xlators/performance/io-threads/src/Makefile.am
+++ b/xlators/performance/io-threads/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = io-threads.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-io_threads_la_LDFLAGS = -module -avoidversion 
+io_threads_la_LDFLAGS = -module -avoid-version 
 
 io_threads_la_SOURCES = io-threads.c
 io_threads_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = io-threads.h iot-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index d3196d50b..8201a95a4 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -23,6 +14,7 @@
 #endif
 
 #include "call-stub.h"
+#include "defaults.h"
 #include "glusterfs.h"
 #include "logging.h"
 #include "dict.h"
@@ -38,17 +30,92 @@ int iot_workers_scale (iot_conf_t *conf);
 int __iot_workers_scale (iot_conf_t *conf);
 struct volume_options options[];
 
+#define IOT_FOP(name, frame, this, args ...)                                   \
+        do {                                                                   \
+                call_stub_t     *__stub     = NULL;                            \
+                int              __ret      = -1;                              \
+                                                                               \
+                __stub = fop_##name##_stub(frame, default_##name##_resume, args);  \
+                if (!__stub) {                                                 \
+                        __ret = -ENOMEM;                                       \
+                        goto out;                                              \
+                }                                                              \
+                                                                               \
+                __ret = iot_schedule (frame, this, __stub);                    \
+                                                                               \
+        out:                                                                   \
+                if (__ret < 0) {                                               \
+                        default_##name##_failure_cbk (frame, -__ret);          \
+                        if (__stub != NULL) {                                  \
+                                call_stub_destroy (__stub);                    \
+                        }                                                      \
+                }                                                              \
+        } while (0)
+
 call_stub_t *
-__iot_dequeue (iot_conf_t *conf, int *pri)
+__iot_dequeue (iot_conf_t *conf, int *pri, struct timespec *sleep)
 {
         call_stub_t  *stub = NULL;
         int           i = 0;
+	struct timeval curtv = {0,}, difftv = {0,};
 
         *pri = -1;
+	sleep->tv_sec = 0;
+	sleep->tv_nsec = 0;
         for (i = 0; i < IOT_PRI_MAX; i++) {
                 if (list_empty (&conf->reqs[i]) ||
                    (conf->ac_iot_count[i] >= conf->ac_iot_limit[i]))
                         continue;
+
+		if (i == IOT_PRI_LEAST) {
+			pthread_mutex_lock(&conf->throttle.lock);
+			if (!conf->throttle.sample_time.tv_sec) {
+				/* initialize */
+				gettimeofday(&conf->throttle.sample_time, NULL);
+			} else {
+				/*
+				 * Maintain a running count of least priority
+				 * operations that are handled over a particular
+				 * time interval. The count is provided via
+				 * state dump and is used as a measure against
+				 * least priority op throttling.
+				 */
+				gettimeofday(&curtv, NULL);
+				timersub(&curtv, &conf->throttle.sample_time,
+					 &difftv);
+				if (difftv.tv_sec >= IOT_LEAST_THROTTLE_DELAY) {
+					conf->throttle.cached_rate =
+						conf->throttle.sample_cnt;
+					conf->throttle.sample_cnt = 0;
+					conf->throttle.sample_time = curtv;
+				}
+
+				/*
+				 * If we're over the configured rate limit,
+				 * provide an absolute time to the caller that
+				 * represents the soonest we're allowed to
+				 * return another least priority request.
+				 */
+				if (conf->throttle.rate_limit &&
+				    conf->throttle.sample_cnt >=
+						conf->throttle.rate_limit) {
+					struct timeval delay;
+					delay.tv_sec = IOT_LEAST_THROTTLE_DELAY;
+					delay.tv_usec = 0;
+
+					timeradd(&conf->throttle.sample_time,
+						 &delay, &curtv);
+					TIMEVAL_TO_TIMESPEC(&curtv, sleep);
+
+					pthread_mutex_unlock(
+						&conf->throttle.lock);
+					break;
+				}
+			}
+			conf->throttle.sample_cnt++;
+			pthread_mutex_unlock(&conf->throttle.lock);
+		}
+
                 stub = list_entry (conf->reqs[i].next, call_stub_t, list);
                 conf->ac_iot_count[i]++;
                 *pri = i;
@@ -59,6 +126,7 @@ __iot_dequeue (iot_conf_t *conf, int *pri)
                 return NULL;
 
         conf->queue_size--;
+        conf->queue_sizes[*pri]--;
         list_del_init (&stub->list);
 
         return stub;
@@ -74,6 +142,7 @@ __iot_enqueue (iot_conf_t *conf, call_stub_t *stub, int pri)
         list_add_tail (&stub->list, &conf->reqs[pri]);
 
         conf->queue_size++;
+        conf->queue_sizes[pri]++;
 
         return;
 }
@@ -90,6 +159,7 @@ iot_worker (void *data)
         int               pri = -1;
         char              timeout = 0;
         char              bye = 0;
+	struct timespec	  sleep = {0,};
 
         conf = data;
         this = conf->this;
@@ -130,7 +200,13 @@ iot_worker (void *data)
                                 }
                         }
 
-                        stub = __iot_dequeue (conf, &pri);
+                        stub = __iot_dequeue (conf, &pri, &sleep);
+			if (!stub && (sleep.tv_sec || sleep.tv_nsec)) {
+				pthread_cond_timedwait(&conf->cond,
+						       &conf->mutex, &sleep);
+				pthread_mutex_unlock(&conf->mutex);
+				continue;
+			}
                 }
                 pthread_mutex_unlock (&conf->mutex);
 
@@ -199,8 +275,9 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
 {
         int             ret = -1;
         iot_pri_t       pri = IOT_PRI_MAX - 1;
+        iot_conf_t      *conf = this->private;
 
-        if (frame->root->pid < 0) {
+        if ((frame->root->pid < GF_CLIENT_PID_MAX) && conf->least_priority) {
                 pri = IOT_PRI_LEAST;
                 goto out;
         }
@@ -252,11 +329,11 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_FSYNCDIR:
         case GF_FOP_XATTROP:
         case GF_FOP_FXATTROP:
-                pri = IOT_PRI_LO;
-                break;
-
         case GF_FOP_RCHECKSUM:
-                pri = IOT_PRI_LEAST;
+	case GF_FOP_FALLOCATE:
+	case GF_FOP_DISCARD:
+        case GF_FOP_ZEROFILL:
+                pri = IOT_PRI_LO;
                 break;
 
         case GF_FOP_NULL:
@@ -270,2041 +347,396 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
                 break;
         }
 out:
-        ret = do_iot_schedule (this->private, stub, pri);
         gf_log (this->name, GF_LOG_DEBUG, "%s scheduled as %s fop",
                 gf_fop_list[stub->fop], iot_get_pri_meaning (pri));
+        ret = do_iot_schedule (this->private, stub, pri);
         return ret;
 }
 
 int
-iot_lookup_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                inode_t *inode, struct iatt *buf, dict_t *xattr,
-                struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xattr,
-                             postparent);
-        return 0;
-}
-
-
-int
-iot_lookup_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    dict_t *xattr_req)
+iot_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_lookup_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lookup,
-                    loc, xattr_req);
-        return 0;
-}
-
-
-int
-iot_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
-{
-        call_stub_t     *stub = NULL;
-        int              ret = -1;
-
-        stub = fop_lookup_stub (frame, iot_lookup_wrapper, loc, xattr_req);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create lookup stub (out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-                STACK_UNWIND_STRICT (lookup, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-        }
-
-        return 0;
-}
-
-
-int
-iot_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preop, struct iatt *postop)
-{
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
-        return 0;
-}
-
-
-int
-iot_setattr_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     struct iatt *stbuf, int32_t valid)
-{
-        STACK_WIND (frame, iot_setattr_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr,
-                    loc, stbuf, valid);
+        IOT_FOP (lookup, frame, this, loc, xdata);
         return 0;
 }
 
 
 int
 iot_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid)
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int              ret = -1;
-
-        stub = fop_setattr_stub (frame, iot_setattr_wrapper, loc, stbuf, valid);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "Cannot create setattr stub"
-                        "(Out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-
-                STACK_UNWIND_STRICT (setattr, frame, -1, -ret, NULL, NULL);
-        }
-
-        return 0;
-}
-
-
-int
-iot_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *preop, struct iatt *postop)
-{
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, preop, postop);
-        return 0;
-}
-
-
-int
-iot_fsetattr_wrapper (call_frame_t *frame, xlator_t *this,
-                      fd_t *fd, struct iatt *stbuf, int32_t valid)
-{
-        STACK_WIND (frame, iot_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid);
+        IOT_FOP (setattr, frame, this, loc, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
 iot_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iatt *stbuf, int32_t valid)
-{
-        call_stub_t     *stub = NULL;
-        int              ret = -1;
-
-        stub = fop_fsetattr_stub (frame, iot_fsetattr_wrapper, fd, stbuf,
-                                  valid);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create fsetattr stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, -ret, NULL, NULL);
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-iot_access_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    int32_t mask)
-{
-        STACK_WIND (frame, iot_access_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->access, loc, mask);
-        return 0;
-}
-
-
-int
-iot_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_access_stub (frame, iot_access_wrapper, loc, mask);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create access stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (access, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, const char *path,
-                  struct iatt *stbuf)
-{
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, stbuf);
-        return 0;
-}
-
-
-int
-iot_readlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      size_t size)
-{
-        STACK_WIND (frame, iot_readlink_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readlink,
-                    loc, size);
-        return 0;
-}
-
-
-int
-iot_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_readlink_stub (frame, iot_readlink_wrapper, loc, size);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create readlink stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (readlink, frame, -1, -ret, NULL, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
+        IOT_FOP (fsetattr, frame, this, fd, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
-iot_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
+iot_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+        IOT_FOP (access, frame, this, loc, mask, xdata);
         return 0;
 }
 
 
 int
-iot_mknod_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-                   dev_t rdev, dict_t *params)
+iot_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mknod, loc, mode, rdev, params);
+        IOT_FOP (readlink, frame, this, loc, size, xdata);
         return 0;
 }
 
 
 int
 iot_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t rdev, dict_t *params)
+           dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_mknod_stub (frame, iot_mknod_wrapper, loc, mode, rdev,
-                               params);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create mknod stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (mknod, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_mkdir_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-        return 0;
-}
-
-
-int
-iot_mkdir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-                   dict_t *params)
-{
-        STACK_WIND (frame, iot_mkdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mkdir, loc, mode, params);
+        IOT_FOP (mknod, frame, this, loc, mode, rdev, umask, xdata);
         return 0;
 }
 
 
 int
 iot_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dict_t *params)
+           mode_t umask, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_mkdir_stub (frame, iot_mkdir_wrapper, loc, mode, params);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create mkdir stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (mkdir, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (mkdir, frame, this, loc, mode, umask, xdata);
         return 0;
 }
 
 
 int
-iot_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-               struct iatt *postparent)
+iot_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent);
-        return 0;
-}
-
-
-int
-iot_rmdir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
-{
-        STACK_WIND (frame, iot_rmdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rmdir, loc, flags);
-        return 0;
-}
-
-
-int
-iot_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_rmdir_stub (frame, iot_rmdir_wrapper, loc, flags);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create rmdir stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (rmdir, frame, -1, -ret, NULL, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_symlink_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-        return 0;
-}
-
-
-int
-iot_symlink_wrapper (call_frame_t *frame, xlator_t *this, const char *linkname,
-                     loc_t *loc, dict_t *params)
-{
-        STACK_WIND (frame, iot_symlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->symlink, linkname, loc, params);
+        IOT_FOP (rmdir, frame, this, loc, flags, xdata);
         return 0;
 }
 
 
 int
 iot_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-             loc_t *loc, dict_t *params)
+             loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_symlink_stub (frame, iot_symlink_wrapper, linkname, loc,
-                                 params);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create symlink stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (symlink, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
+        IOT_FOP (symlink, frame, this, linkname, loc, umask, xdata);
         return 0;
 }
 
 
 int
-iot_rename_cbk (call_frame_t *frame, void * cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent)
+iot_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent);
-        return 0;
-}
-
-
-int
-iot_rename_wrapper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                    loc_t *newloc)
-{
-        STACK_WIND (frame, iot_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rename, oldloc, newloc);
-        return 0;
-}
-
-
-int
-iot_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_rename_stub (frame, iot_rename_wrapper, oldloc, newloc);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "cannot create rename stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (rename, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL, NULL);
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
+        IOT_FOP (rename, frame, this, oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
-iot_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, fd_t *fd)
-{
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-
-int
-iot_open_wrapper (call_frame_t * frame, xlator_t * this, loc_t *loc,
-                  int32_t flags, fd_t * fd, int32_t wbflags)
-{
-	STACK_WIND (frame, iot_open_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->open, loc, flags, fd, wbflags);
-	return 0;
-}
-
-
-int
 iot_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+          fd_t *fd, dict_t *xdata)
 {
-        call_stub_t	*stub = NULL;
-        int             ret = -1;
-
-        stub = fop_open_stub (frame, iot_open_wrapper, loc, flags, fd, wbflags);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create open call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-	ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (open, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
-	return 0;
-}
-
-
-int
-iot_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                struct iatt *stbuf, struct iatt *preparent,
-                struct iatt *postparent)
-{
-	STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, stbuf,
-                             preparent, postparent);
-	return 0;
-}
-
-
-int
-iot_create_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
-{
-	STACK_WIND (frame, iot_create_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd, params);
-	return 0;
+        IOT_FOP (open, frame, this, loc, flags, fd, xdata);
+        return 0;
 }
 
 
 int
 iot_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-            mode_t mode, fd_t *fd, dict_t *params)
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_create_stub (frame, iot_create_wrapper, loc, flags, mode,
-                                fd, params);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create \"create\" call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (create, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
+        IOT_FOP (create, frame, this, loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
 
 int
-iot_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iovec *vector,
-               int32_t count, struct iatt *stbuf, struct iobref *iobref)
-{
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
-
-	return 0;
-}
-
-
-int
-iot_readv_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                   off_t offset, uint32_t flags)
-{
-	STACK_WIND (frame, iot_readv_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readv,
-		    fd, size, offset, flags);
-	return 0;
-}
-
-
-int
 iot_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-           off_t offset, uint32_t flags)
-{
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-	stub = fop_readv_stub (frame, iot_readv_wrapper, fd, size, offset, flags);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"cannot create readv call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (readv, frame, -1, -ret, NULL, -1, NULL,
-                                     NULL);
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-	return 0;
-}
-
-
-int
-iot_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
-	return 0;
-}
-
-
-int
-iot_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-	STACK_WIND (frame, iot_flush_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->flush,
-		    fd);
-	return 0;
-}
-
-
-int
-iot_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-	stub = fop_flush_stub (frame, iot_flush_wrapper, fd);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create flush_cbk call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (flush, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-	return 0;
-}
-
-
-int
-iot_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
-{
-	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-
-int
-iot_fsync_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   int32_t datasync)
-{
-	STACK_WIND (frame, iot_fsync_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->fsync,
-		    fd, datasync);
-	return 0;
-}
-
-
-int
-iot_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
+           off_t offset, uint32_t flags, dict_t *xdata)
 {
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-	stub = fop_fsync_stub (frame, iot_fsync_wrapper, fd, datasync);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create fsync_cbk call stub"
-                        "(out of memory)");
-                ret = -1;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (fsync, frame, -1, -ret, NULL, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-	return 0;
-}
-
-
-int
-iot_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
-{
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-
-int
-iot_writev_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iovec *vector, int32_t count,
-                    off_t offset, uint32_t flags, struct iobref *iobref)
-{
-	STACK_WIND (frame, iot_writev_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->writev,
-		    fd, vector, count, offset, flags, iobref);
-	return 0;
-}
-
-
-int
-iot_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref)
-{
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-	stub = fop_writev_stub (frame, iot_writev_wrapper,
-				fd, vector, count, offset, flags, iobref);
-
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create writev call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (writev, frame, -1, -ret, NULL, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
-	return 0;
-}
-
-
-int32_t
-iot_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *flock)
-{
-	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock);
-	return 0;
-}
-
-
-int
-iot_lk_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                int32_t cmd, struct gf_flock *flock)
-{
-	STACK_WIND (frame, iot_lk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lk,
-		    fd, cmd, flock);
-	return 0;
-}
-
-
-int
-iot_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	struct gf_flock *flock)
-{
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-	stub = fop_lk_stub (frame, iot_lk_wrapper, fd, cmd, flock);
-
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create fop_lk call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (lk, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-	return 0;
-}
-
-
-int
-iot_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-int
-iot_stat_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-	STACK_WIND (frame, iot_stat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->stat,
-		    loc);
-	return 0;
-}
-
-
-int
-iot_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-        stub = fop_stat_stub (frame, iot_stat_wrapper, loc);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create fop_stat call stub"
-                        "(out of memory)");
-                ret = -1;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (stat, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-	return 0;
-}
-
-
-int
-iot_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-int
-iot_fstat_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-	STACK_WIND (frame, iot_fstat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fstat,
-		    fd);
-	return 0;
-}
-
-
-int
-iot_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-	stub = fop_fstat_stub (frame, iot_fstat_wrapper, fd);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create fop_fstat call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (fstat, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-	return 0;
-}
-
-
-int
-iot_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-	STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
-	return 0;
-}
-
-
-int
-iot_truncate_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      off_t offset)
-{
-	STACK_WIND (frame, iot_truncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset);
-	return 0;
-}
-
-
-int
-iot_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
-{
-	call_stub_t *stub;
-        int         ret = -1;
-
-        stub = fop_truncate_stub (frame, iot_truncate_wrapper, loc, offset);
-
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create fop_stat call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (truncate, frame, -1, -ret, NULL, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
-	return 0;
-}
-
-
-int
-iot_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-	STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
-	return 0;
-}
-
-
-int
-iot_ftruncate_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       off_t offset)
-{
-	STACK_WIND (frame, iot_ftruncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset);
-	return 0;
-}
-
-
-int
-iot_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
-{
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-	stub = fop_ftruncate_stub (frame, iot_ftruncate_wrapper, fd, offset);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create fop_ftruncate call stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (ftruncate, frame, -1, -ret, NULL, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-	return 0;
-}
-
-
-
-int
-iot_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                struct iatt *postparent)
-{
-	STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent);
-	return 0;
-}
-
-
-int
-iot_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-	STACK_WIND (frame, iot_unlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
-	return 0;
-}
-
-
-int
-iot_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-	call_stub_t *stub = NULL;
-        int         ret = -1;
-
-	stub = fop_unlink_stub (frame, iot_unlink_wrapper, loc);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create fop_unlink call stub"
-                        "(out of memory)");
-                ret = -1;
-                goto out;
-	}
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-		STACK_UNWIND_STRICT (unlink, frame, -1, -ret, NULL, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
-	return 0;
-}
-
-
-int
-iot_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, inode_t *inode,
-              struct iatt *buf, struct iatt *preparent, struct iatt *postparent)
-{
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+        IOT_FOP (readv, frame, this, fd, size, offset, flags, xdata);
         return 0;
 }
 
 
 int
-iot_link_wrapper (call_frame_t *frame, xlator_t *this, loc_t *old, loc_t *new)
+iot_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_link_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->link, old, new);
-
-        return 0;
-}
-
-
-int
-iot_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_link_stub (frame, iot_link_wrapper, oldloc, newloc);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create link stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (link, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (flush, frame, this, fd, xdata);
         return 0;
 }
 
 
 int
-iot_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
+iot_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+           dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        IOT_FOP (fsync, frame, this, fd, datasync, xdata);
         return 0;
 }
 
 
 int
-iot_opendir_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+iot_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+            struct iovec *vector, int32_t count, off_t offset,
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_opendir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->opendir, loc, fd);
+        IOT_FOP (writev, frame, this, fd, vector, count, offset,
+                 flags, iobref, xdata);
         return 0;
 }
 
 
 int
-iot_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+iot_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+	struct gf_flock *flock, dict_t *xdata)
 {
-        call_stub_t     *stub  = NULL;
-        int             ret = -1;
-
-        stub = fop_opendir_stub (frame, iot_opendir_wrapper, loc, fd);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create opendir stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (opendir, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (lk, frame, this, fd, cmd, flock, xdata);
         return 0;
 }
 
 
 int
-iot_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+iot_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+        IOT_FOP (stat, frame, this, loc, xdata);
         return 0;
 }
 
 
 int
-iot_fsyncdir_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      int datasync)
+iot_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_fsyncdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsyncdir, fd, datasync);
+        IOT_FOP (fstat, frame, this, fd, xdata);
         return 0;
 }
 
 
 int
-iot_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+iot_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_fsyncdir_stub (frame, iot_fsyncdir_wrapper, fd, datasync);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create fsyncdir stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (truncate, frame, this, loc, offset, xdata);
         return 0;
 }
 
 
 int
-iot_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+iot_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        IOT_FOP (ftruncate, frame, this, fd, offset, xdata);
         return 0;
 }
 
 
-int
-iot_statfs_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        STACK_WIND (frame, iot_statfs_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->statfs, loc);
-        return 0;
-}
-
 
 int
-iot_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+iot_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+            dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int              ret = -1;
-
-        stub = fop_statfs_stub (frame, iot_statfs_wrapper, loc);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create statfs stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (statfs, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (unlink, frame, this, loc, xflag, xdata);
         return 0;
 }
 
 
 int
-iot_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+iot_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        IOT_FOP (link, frame, this, oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
-iot_setxattr_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      dict_t *dict, int32_t flags)
+iot_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
-        STACK_WIND (frame, iot_setxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setxattr, loc, dict, flags);
+        IOT_FOP (opendir, frame, this, loc, fd, xdata);
         return 0;
 }
 
 
 int
-iot_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-              int32_t flags)
+iot_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+              dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int              ret = -1;
-
-        stub = fop_setxattr_stub (frame, iot_setxattr_wrapper, loc, dict,
-                                  flags);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create setxattr stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (setxattr, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (fsyncdir, frame, this, fd, datasync, xdata);
         return 0;
 }
 
 
 int
-iot_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
+iot_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        IOT_FOP (statfs, frame, this, loc, xdata);
         return 0;
 }
 
 
 int
-iot_getxattr_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      const char *name)
+iot_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+              int32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_getxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->getxattr, loc, name);
+        IOT_FOP (setxattr, frame, this, loc, dict, flags, xdata);
         return 0;
 }
 
 
 int
 iot_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              const char *name)
+              const char *name, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int              ret = -1;
-
-        stub = fop_getxattr_stub (frame, iot_getxattr_wrapper, loc, name);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create getxattr stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-
-int
-iot_fgetxattr_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       const char *name)
-{
-        STACK_WIND (frame, iot_fgetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fgetxattr, fd, name);
+        IOT_FOP (getxattr, frame, this, loc, name, xdata);
         return 0;
 }
 
 
 int
 iot_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               const char *name)
+               const char *name, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_fgetxattr_stub (frame, iot_fgetxattr_wrapper, fd, name);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create fgetxattr stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-iot_fsetxattr_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       dict_t *dict, int32_t flags)
-{
-        STACK_WIND (frame, iot_fsetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags);
+        IOT_FOP (fgetxattr, frame, this, fd, name, xdata);
         return 0;
 }
 
 
 int
 iot_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-               int32_t flags)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_fsetxattr_stub (frame, iot_fsetxattr_wrapper, fd, dict,
-                                        flags);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create fsetxattr stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-iot_removexattr_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         const char *name)
+               int32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_removexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->removexattr, loc, name);
+        IOT_FOP (fsetxattr, frame, this, fd, dict, flags, xdata);
         return 0;
 }
 
 
 int
 iot_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_removexattr_stub (frame, iot_removexattr_wrapper, loc,
-                                     name);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR,"cannot get removexattr fop"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (removexattr, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (removexattr, frame, this, loc, name, xdata);
         return 0;
 }
 
 int
-iot_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-iot_fremovexattr_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          const char *name)
-{
-        STACK_WIND (frame, iot_fremovexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fremovexattr, fd, name);
-        return 0;
-}
-
-
-int
 iot_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_fremovexattr_stub (frame, iot_fremovexattr_wrapper, fd,
-                                      name);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR,"cannot get fremovexattr fop"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (fremovexattr, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
-        return 0;
-}
-
-
-int
-iot_readdirp_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      size_t size, off_t offset, dict_t *dict)
-{
-        STACK_WIND (frame, iot_readdirp_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readdirp, fd, size, offset, dict);
+        IOT_FOP (fremovexattr, frame, this, fd, name, xdata);
         return 0;
 }
 
 
 int
 iot_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, dict_t *dict)
+              off_t offset, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_readdirp_stub (frame, iot_readdirp_wrapper, fd, size,
-                                  offset, dict);
-        if (!stub) {
-                gf_log (this->private, GF_LOG_ERROR,"cannot get readdir stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (readdirp, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int
-iot_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
-        return 0;
-}
-
-
-int
-iot_readdir_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     size_t size, off_t offset)
-{
-        STACK_WIND (frame, iot_readdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readdir, fd, size, offset);
+        IOT_FOP (readdirp, frame, this, fd, size, offset, xdata);
         return 0;
 }
 
 
 int
 iot_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset)
+             off_t offset, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_readdir_stub (frame, iot_readdir_wrapper, fd, size, offset);
-        if (!stub) {
-                gf_log (this->private, GF_LOG_ERROR,"cannot get readdir stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (readdir, frame, this, fd, size, offset, xdata);
         return 0;
 }
 
 int
-iot_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-iot_inodelk_wrapper (call_frame_t *frame, xlator_t *this, const char *volume,
-                     loc_t *loc, int32_t cmd, struct gf_flock *lock)
-{
-        STACK_WIND (frame, iot_inodelk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk, volume, loc, cmd, lock);
-        return 0;
-}
-
-
-int
 iot_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
+             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock,
+             dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_inodelk_stub (frame, iot_inodelk_wrapper,
-                                 volume, loc, cmd, lock);
-        if (!stub) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (inodelk, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (inodelk, frame, this, volume, loc, cmd, lock, xdata);
         return 0;
 }
 
 int
-iot_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-iot_finodelk_wrapper (call_frame_t *frame, xlator_t *this,
-                      const char *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *lock)
-{
-        STACK_WIND (frame, iot_finodelk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->finodelk, volume, fd, cmd, lock);
-        return 0;
-}
-
-
-int
 iot_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
+              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock,
+              dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_finodelk_stub (frame, iot_finodelk_wrapper,
-                                  volume, fd, cmd, lock);
-        if (!stub) {
-                gf_log (this->private, GF_LOG_ERROR,"cannot get finodelk stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (finodelk, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (finodelk, frame, this, volume, fd, cmd, lock, xdata);
         return 0;
 }
 
 int
-iot_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-iot_entrylk_wrapper (call_frame_t *frame, xlator_t *this,
-                     const char *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd cmd, entrylk_type type)
-{
-        STACK_WIND (frame, iot_entrylk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
-        return 0;
-}
-
-
-int
 iot_entrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type)
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_entrylk_stub (frame, iot_entrylk_wrapper,
-                                 volume, loc, basename, cmd, type);
-        if (!stub) {
-                gf_log (this->private, GF_LOG_ERROR,"cannot get entrylk stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (entrylk, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-int
-iot_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
+        IOT_FOP (entrylk, frame, this, volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
-
-int
-iot_fentrylk_wrapper (call_frame_t *frame, xlator_t *this,
-                      const char *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd cmd, entrylk_type type)
-{
-        STACK_WIND (frame, iot_fentrylk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type);
-        return 0;
-}
-
-
 int
 iot_fentrylk (call_frame_t *frame, xlator_t *this,
               const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type)
+              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_fentrylk_stub (frame, iot_fentrylk_wrapper,
-                                  volume, fd, basename, cmd, type);
-        if (!stub) {
-                gf_log (this->private, GF_LOG_ERROR,"cannot get fentrylk stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, -ret);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (fentrylk, frame, this, volume, fd, basename, cmd, type, xdata);
         return 0;
 }
 
 
 int
-iot_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xattr)
+iot_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, xattr);
+        IOT_FOP (xattrop, frame, this, loc, optype, xattr, xdata);
         return 0;
 }
 
 
 int
-iot_xattrop_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     gf_xattrop_flags_t optype, dict_t *xattr)
+iot_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_xattrop_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr);
+        IOT_FOP (fxattrop, frame, this, fd, optype, xattr, xdata);
         return 0;
 }
 
 
-int
-iot_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr)
+int32_t
+iot_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               int32_t len, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_xattrop_stub (frame, iot_xattrop_wrapper, loc, optype,
-                                        xattr);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create xattrop stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (xattrop, frame, -1, -ret, NULL);
-
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
+        IOT_FOP (rchecksum, frame, this, fd, offset, len, xdata);
         return 0;
 }
 
-
 int
-iot_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xattr)
+iot_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	      off_t offset, size_t len, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, xattr);
+        IOT_FOP (fallocate, frame, this, fd, mode, offset, len, xdata);
         return 0;
 }
 
 int
-iot_fxattrop_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      gf_xattrop_flags_t optype, dict_t *xattr)
+iot_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_fxattrop_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr);
+        IOT_FOP (discard, frame, this, fd, offset, len, xdata);
         return 0;
 }
 
-
 int
-iot_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_fxattrop_stub (frame, iot_fxattrop_wrapper, fd, optype,
-                                        xattr);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create fxattrop stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, -ret, NULL);
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-        return 0;
-}
-
-
-int32_t
-iot_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                   uint8_t *strong_checksum)
-{
-        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum);
-        return 0;
-}
-
-
-int32_t
-iot_rchecksum_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       off_t offset, int32_t len)
+iot_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
 {
-        STACK_WIND (frame, iot_rchecksum_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum, fd, offset, len);
-        return 0;
-}
-
-
-int32_t
-iot_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               int32_t len)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -1;
-
-        stub = fop_rchecksum_stub (frame, iot_rchecksum_wrapper, fd, offset,
-                                   len);
-        if (!stub) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot create rchecksum stub"
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = iot_schedule (frame, this, stub);
-out:
-        if (ret < 0) {
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, -ret, -1, NULL);
-                if (stub != NULL) {
-                        call_stub_destroy (stub);
-                }
-        }
-
+        IOT_FOP (zerofill, frame, this, fd, offset, len, xdata);
         return 0;
 }
 
@@ -2312,20 +744,19 @@ out:
 int
 __iot_workers_scale (iot_conf_t *conf)
 {
-        int       log2 = 0;
         int       scale = 0;
         int       diff = 0;
         pthread_t thread;
         int       ret = 0;
+        int       i = 0;
 
-        log2 = log_base2 (conf->queue_size);
+        for (i = 0; i < IOT_PRI_MAX; i++)
+                scale += min (conf->queue_sizes[i], conf->ac_iot_limit[i]);
 
-        scale = log2;
-
-        if (log2 < IOT_MIN_THREADS)
+        if (scale < IOT_MIN_THREADS)
                 scale = IOT_MIN_THREADS;
 
-        if (log2 > conf->max_count)
+        if (scale > conf->max_count)
                 scale = conf->max_count;
 
         if (conf->curr_count < scale) {
@@ -2335,7 +766,7 @@ __iot_workers_scale (iot_conf_t *conf)
         while (diff) {
                 diff --;
 
-                ret = pthread_create (&thread, &conf->w_attr, iot_worker, conf);
+                ret = gf_thread_create (&thread, &conf->w_attr, iot_worker, conf);
                 if (ret == 0) {
                         conf->curr_count++;
                         gf_log (conf->this->name, GF_LOG_DEBUG,
@@ -2376,13 +807,24 @@ set_stack_size (iot_conf_t *conf)
 {
         int     err = 0;
         size_t  stacksize = IOT_THREAD_STACK_SIZE;
+        xlator_t *this = NULL;
+
+        this = THIS;
 
         pthread_attr_init (&conf->w_attr);
         err = pthread_attr_setstacksize (&conf->w_attr, stacksize);
         if (err == EINVAL) {
-                gf_log (conf->this->name, GF_LOG_WARNING,
-                        "Using default thread stack size");
+                err = pthread_attr_getstacksize (&conf->w_attr, &stacksize);
+                if (!err)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Using default thread stack size %zd",
+                                stacksize);
+                else
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Using default thread stack size");
         }
+
+        conf->stack_size = stacksize;
 }
 
 
@@ -2405,6 +847,44 @@ mem_acct_init (xlator_t *this)
         return ret;
 }
 
+int
+iot_priv_dump (xlator_t *this)
+{
+        iot_conf_t     *conf   =   NULL;
+        char           key_prefix[GF_DUMP_MAX_BUF_LEN];
+
+        if (!this)
+                return 0;
+
+        conf = this->private;
+        if (!conf)
+                return 0;
+
+        snprintf (key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type,
+                  this->name);
+
+        gf_proc_dump_add_section(key_prefix);
+
+        gf_proc_dump_write("maximum_threads_count", "%d", conf->max_count);
+        gf_proc_dump_write("current_threads_count", "%d", conf->curr_count);
+        gf_proc_dump_write("sleep_count", "%d", conf->sleep_count);
+        gf_proc_dump_write("idle_time", "%d", conf->idle_time);
+        gf_proc_dump_write("stack_size", "%zd", conf->stack_size);
+        gf_proc_dump_write("high_priority_threads", "%d",
+                           conf->ac_iot_limit[IOT_PRI_HI]);
+        gf_proc_dump_write("normal_priority_threads", "%d",
+                           conf->ac_iot_limit[IOT_PRI_NORMAL]);
+        gf_proc_dump_write("low_priority_threads", "%d",
+                           conf->ac_iot_limit[IOT_PRI_LO]);
+        gf_proc_dump_write("least_priority_threads", "%d",
+                           conf->ac_iot_limit[IOT_PRI_LEAST]);
+
+	gf_proc_dump_write("cached least rate", "%u",
+			   conf->throttle.cached_rate);
+	gf_proc_dump_write("least rate limit", "%u", conf->throttle.rate_limit);
+
+        return 0;
+}
 
 int
 reconfigure (xlator_t *this, dict_t *options)
@@ -2431,6 +911,11 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("least-prio-threads",
                           conf->ac_iot_limit[IOT_PRI_LEAST], options, int32,
                           out);
+        GF_OPTION_RECONF ("enable-least-priority", conf->least_priority,
+                          options, bool, out);
+
+	GF_OPTION_RECONF("least-rate-limit", conf->throttle.rate_limit, options,
+			 int32, out);
 
 	ret = 0;
 out:
@@ -2441,9 +926,9 @@ out:
 int
 init (xlator_t *this)
 {
-        iot_conf_t      *conf = NULL;
-        int              ret = -1;
-        int              i = 0;
+        iot_conf_t *conf = NULL;
+        int         ret  = -1;
+        int         i    = 0;
 
 	if (!this->children || this->children->next) {
 		gf_log ("io-threads", GF_LOG_ERROR,
@@ -2493,6 +978,16 @@ init (xlator_t *this)
                         conf->ac_iot_limit[IOT_PRI_LEAST], int32, out);
 
         GF_OPTION_INIT ("idle-time", conf->idle_time, int32, out);
+        GF_OPTION_INIT ("enable-least-priority", conf->least_priority,
+                        bool, out);
+
+	GF_OPTION_INIT("least-rate-limit", conf->throttle.rate_limit, int32,
+		       out);
+        if ((ret = pthread_mutex_init(&conf->throttle.lock, NULL)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_mutex_init failed (%d)", ret);
+                goto out;
+        }
 
         conf->this = this;
 
@@ -2505,13 +1000,15 @@ init (xlator_t *this)
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "cannot initialize worker threads, exiting init");
-                GF_FREE (conf);
                 goto out;
         }
 
 	this->private = conf;
         ret = 0;
 out:
+        if (ret)
+                GF_FREE (conf);
+
 	return ret;
 }
 
@@ -2527,6 +1024,9 @@ fini (xlator_t *this)
 	return;
 }
 
+struct xlator_dumpops dumpops = {
+        .priv    = iot_priv_dump,
+};
 
 struct xlator_fops fops = {
 	.open        = iot_open,
@@ -2570,10 +1070,12 @@ struct xlator_fops fops = {
         .xattrop     = iot_xattrop,
 	.fxattrop    = iot_fxattrop,
         .rchecksum   = iot_rchecksum,
+	.fallocate   = iot_fallocate,
+	.discard     = iot_discard,
+        .zerofill    = iot_zerofill,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
 	{ .key  = {"thread-count"},
@@ -2620,12 +1122,25 @@ struct volume_options options[] = {
           .description = "Max number of threads in IO threads translator which "
                          "perform least priority IO operations at a given time"
 	},
+        { .key  = {"enable-least-priority"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Enable/Disable least priority"
+        },
         {.key   = {"idle-time"},
          .type  = GF_OPTION_TYPE_INT,
          .min   = 1,
          .max   = 0x7fffffff,
          .default_value = "120",
         },
+	{.key	= {"least-rate-limit"},
+	 .type	= GF_OPTION_TYPE_INT,
+	 .min	= 0,
+         .max	= INT_MAX,
+	 .default_value = "0",
+	 .description = "Max number of least priority operations to handle "
+			"per-second"
+	},
 	{ .key  = {NULL},
         },
 };
diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h
index d09fec94d..1a9dee9ae 100644
--- a/xlators/performance/io-threads/src/io-threads.h
+++ b/xlators/performance/io-threads/src/io-threads.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __IOT_H
@@ -37,6 +28,7 @@
 #include "locking.h"
 #include "iot-mem-types.h"
 #include <semaphore.h>
+#include "statedump.h"
 
 
 struct iot_conf;
@@ -61,6 +53,14 @@ typedef enum {
         IOT_PRI_MAX,
 } iot_pri_t;
 
+#define IOT_LEAST_THROTTLE_DELAY 1	/* sample interval in seconds */
+struct iot_least_throttle {
+	struct timeval	sample_time;	/* timestamp of current sample */
+	uint32_t	sample_cnt;	/* sample count for active interval */
+	uint32_t	cached_rate;	/* the most recently measured rate */
+	int32_t		rate_limit;	/* user-specified rate limit */
+	pthread_mutex_t	lock;
+};
 
 struct iot_conf {
         pthread_mutex_t      mutex;
@@ -76,10 +76,15 @@ struct iot_conf {
 
         int32_t              ac_iot_limit[IOT_PRI_MAX];
         int32_t              ac_iot_count[IOT_PRI_MAX];
+        int                  queue_sizes[IOT_PRI_MAX];
         int                  queue_size;
         pthread_attr_t       w_attr;
+        gf_boolean_t         least_priority; /*Enable/Disable least-priority */
 
         xlator_t            *this;
+        size_t              stack_size;
+
+	struct iot_least_throttle throttle;
 };
 
 typedef struct iot_conf iot_conf_t;
diff --git a/xlators/performance/io-threads/src/iot-mem-types.h b/xlators/performance/io-threads/src/iot-mem-types.h
index 14400cd08..4fa8302d1 100644
--- a/xlators/performance/io-threads/src/iot-mem-types.h
+++ b/xlators/performance/io-threads/src/iot-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
diff --git a/xlators/performance/md-cache/src/Makefile.am b/xlators/performance/md-cache/src/Makefile.am
index 150e7243e..8c9f5a858 100644
--- a/xlators/performance/md-cache/src/Makefile.am
+++ b/xlators/performance/md-cache/src/Makefile.am
@@ -1,15 +1,17 @@
 xlator_LTLIBRARIES = md-cache.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-md_cache_la_LDFLAGS = -module -avoidversion
+md_cache_la_LDFLAGS = -module -avoid-version
 
 md_cache_la_SOURCES = md-cache.c
 md_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = md-cache-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/rbtree -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(CONTRIBDIR)/rbtree
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
 
diff --git a/xlators/performance/md-cache/src/md-cache-mem-types.h b/xlators/performance/md-cache/src/md-cache-mem-types.h
index f09e88257..6634cf962 100644
--- a/xlators/performance/md-cache/src/md-cache-mem-types.h
+++ b/xlators/performance/md-cache/src/md-cache-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c
index 5fa6e214b..ef156e309 100644
--- a/xlators/performance/md-cache/src/md-cache.c
+++ b/xlators/performance/md-cache/src/md-cache.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -27,6 +18,7 @@
 #include "dict.h"
 #include "xlator.h"
 #include "md-cache-mem-types.h"
+#include "glusterfs-acl.h"
 #include <assert.h>
 #include <sys/time.h>
 
@@ -39,32 +31,35 @@
 
 struct mdc_conf {
 	int  timeout;
+	gf_boolean_t cache_posix_acl;
+	gf_boolean_t cache_selinux;
+	gf_boolean_t force_readdirp;
 };
 
 
-struct mdc_key {
+static struct mdc_key {
 	const char *name;
 	int         load;
 	int         check;
 } mdc_keys[] = {
 	{
-		.name = "system.posix_acl_access",
-		.load = 1,
+		.name = POSIX_ACL_ACCESS_XATTR,
+		.load = 0,
 		.check = 1,
 	},
 	{
-		.name = "system.posix_acl_default",
-		.load = 1,
+		.name = POSIX_ACL_DEFAULT_XATTR,
+		.load = 0,
 		.check = 1,
 	},
 	{
-		.name = "security.selinux",
-		.load = 1,
+		.name = GF_SELINUX_XATTR_KEY,
+		.load = 0,
 		.check = 1,
 	},
 	{
 		.name = "security.capability",
-		.load = 1,
+		.load = 0,
 		.check = 1,
 	},
 	{
@@ -72,7 +67,11 @@ struct mdc_key {
 		.load = 0,
 		.check = 1,
 	},
-	{},
+        {
+                .name = NULL,
+                .load = 0,
+                .check = 0,
+        }
 };
 
 
@@ -134,6 +133,7 @@ struct mdc_local {
         loc_t   loc2;
         fd_t   *fd;
         char   *linkname;
+	char   *key;
         dict_t *xattr;
 };
 
@@ -176,7 +176,7 @@ __mdc_inode_ctx_set (xlator_t *this, inode_t *inode, struct md_cache *mdc)
         uint64_t         mdc_int = 0;
 
 	mdc_int = (long) mdc;
-	ret = __inode_ctx_set2 (inode, this, &mdc_int, 0);
+	ret = __inode_ctx_set (inode, this, &mdc_int);
 
 	return ret;
 }
@@ -229,8 +229,9 @@ mdc_local_wipe (xlator_t *this, mdc_local_t *local)
         if (local->fd)
                 fd_unref (local->fd);
 
-        if (local->linkname)
-                GF_FREE (local->linkname);
+        GF_FREE (local->linkname);
+
+        GF_FREE (local->key);
 
         if (local->xattr)
                 dict_unref (local->xattr);
@@ -256,8 +257,7 @@ mdc_inode_wipe (xlator_t *this, inode_t *inode)
         if (mdc->xattr)
                 dict_unref (mdc->xattr);
 
-        if (mdc->linkname)
-                GF_FREE (mdc->linkname);
+        GF_FREE (mdc->linkname);
 
         GF_FREE (mdc);
 
@@ -315,7 +315,7 @@ is_md_cache_iatt_valid (xlator_t *this, struct md_cache *mdc)
 
         LOCK (&mdc->lock);
         {
-                if (now > (mdc->ia_time + conf->timeout))
+                if (now >= (mdc->ia_time + conf->timeout))
                         ret = _gf_false;
         }
         UNLOCK (&mdc->lock);
@@ -337,7 +337,7 @@ is_md_cache_xatt_valid (xlator_t *this, struct md_cache *mdc)
 
         LOCK (&mdc->lock);
         {
-                if (now > (mdc->xa_time + conf->timeout))
+                if (now >= (mdc->xa_time + conf->timeout))
                         ret = _gf_false;
         }
         UNLOCK (&mdc->lock);
@@ -385,7 +385,8 @@ mdc_to_iatt (struct md_cache *mdc, struct iatt *iatt)
 
 
 int
-mdc_inode_iatt_set (xlator_t *this, inode_t *inode, struct iatt *iatt)
+mdc_inode_iatt_set_validate(xlator_t *this, inode_t *inode, struct iatt *prebuf,
+			    struct iatt *iatt)
 {
         int              ret = -1;
         struct md_cache *mdc = NULL;
@@ -401,6 +402,19 @@ mdc_inode_iatt_set (xlator_t *this, inode_t *inode, struct iatt *iatt)
                         goto unlock;
                 }
 
+		/*
+		 * Invalidate the inode if the mtime or ctime has changed
+		 * and the prebuf doesn't match the value we have cached.
+		 * TODO: writev returns with a NULL iatt due to
+		 * performance/write-behind, causing invalidation on writes.
+		 */
+		if (IA_ISREG(inode->ia_type) &&
+		    ((iatt->ia_mtime != mdc->md_mtime) ||
+		    (iatt->ia_ctime != mdc->md_ctime)))
+			if (!prebuf || (prebuf->ia_ctime != mdc->md_ctime) ||
+			    (prebuf->ia_mtime != mdc->md_mtime))
+				inode_invalidate(inode);
+
                 mdc_from_iatt (mdc, iatt);
 
                 time (&mdc->ia_time);
@@ -412,6 +426,10 @@ out:
         return ret;
 }
 
+int mdc_inode_iatt_set(xlator_t *this, inode_t *inode, struct iatt *iatt)
+{
+	return mdc_inode_iatt_set_validate(this, inode, NULL, iatt);
+}
 
 int
 mdc_inode_iatt_get (xlator_t *this, inode_t *inode, struct iatt *iatt)
@@ -441,12 +459,71 @@ out:
         return ret;
 }
 
+struct updatedict {
+	dict_t *dict;
+	int ret;
+};
+
+static int
+updatefn(dict_t *dict, char *key, data_t *value, void *data)
+{
+	struct updatedict *u = data;
+	const char *mdc_key;
+	int i = 0;
+
+	for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
+		if (!mdc_keys[i].check)
+			continue;
+		if (strcmp(mdc_key, key))
+			continue;
+
+		if (!u->dict) {
+			u->dict = dict_new();
+			if (!u->dict) {
+				u->ret = -1;
+				return -1;
+			}
+		}
+
+		if (dict_set(u->dict, key, value) < 0) {
+			u->ret = -1;
+			return -1;
+		}
+
+		break;
+	}
+        return 0;
+}
+
+static int
+mdc_dict_update(dict_t **tgt, dict_t *src)
+{
+	struct updatedict u = {
+		.dict = *tgt,
+		.ret = 0,
+	};
+
+	dict_foreach(src, updatefn, &u);
+
+	if (*tgt)
+		return u.ret;
+
+	if ((u.ret < 0) && u.dict) {
+		dict_unref(u.dict);
+		return u.ret;
+	}
+
+	*tgt = u.dict;
+
+	return u.ret;
+}
 
 int
 mdc_inode_xatt_set (xlator_t *this, inode_t *inode, dict_t *dict)
 {
         int              ret = -1;
         struct md_cache *mdc = NULL;
+	dict_t		*newdict = NULL;
 
         mdc = mdc_inode_prep (this, inode);
         if (!mdc)
@@ -457,10 +534,19 @@ mdc_inode_xatt_set (xlator_t *this, inode_t *inode, dict_t *dict)
 
         LOCK (&mdc->lock);
         {
-                if (mdc->xattr)
+                if (mdc->xattr) {
                         dict_unref (mdc->xattr);
+			mdc->xattr = NULL;
+		}
+
+		ret = mdc_dict_update(&newdict, dict);
+		if (ret < 0) {
+			UNLOCK(&mdc->lock);
+			goto out;
+		}
 
-                mdc->xattr = dict_ref (dict);
+		if (newdict)
+			mdc->xattr = newdict;
 
                 time (&mdc->xa_time);
         }
@@ -486,12 +572,11 @@ mdc_inode_xatt_update (xlator_t *this, inode_t *inode, dict_t *dict)
 
         LOCK (&mdc->lock);
         {
-                if (!mdc->xattr)
-                        mdc->xattr = dict_ref (dict);
-                else
-                        dict_copy (dict, mdc->xattr);
-
-                mdc->xattr = dict_ref (dict);
+		ret = mdc_dict_update(&mdc->xattr, dict);
+		if (ret < 0) {
+			UNLOCK(&mdc->lock);
+			goto out;
+		}
 
                 time (&mdc->xa_time);
         }
@@ -504,6 +589,31 @@ out:
 
 
 int
+mdc_inode_xatt_unset (xlator_t *this, inode_t *inode, char *name)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        mdc = mdc_inode_prep (this, inode);
+        if (!mdc)
+                goto out;
+
+        if (!name)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+		dict_del (mdc->xattr, name);
+        }
+        UNLOCK (&mdc->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
 mdc_inode_xatt_get (xlator_t *this, inode_t *inode, dict_t **dict)
 {
         int              ret = -1;
@@ -517,13 +627,15 @@ mdc_inode_xatt_get (xlator_t *this, inode_t *inode, dict_t **dict)
 
         LOCK (&mdc->lock);
         {
+                ret = 0;
+		/* Missing xattr only means no keys were there, i.e
+		   a negative cache for the "loaded" keys
+		*/
                 if (!mdc->xattr)
                         goto unlock;
 
                 if (dict)
                         *dict = dict_ref (mdc->xattr);
-
-                ret = 0;
         }
 unlock:
         UNLOCK (&mdc->lock);
@@ -533,6 +645,46 @@ out:
 }
 
 
+int
+mdc_inode_iatt_invalidate (xlator_t *this, inode_t *inode)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+		mdc->ia_time = 0;
+        }
+        UNLOCK (&mdc->lock);
+
+out:
+        return ret;
+}
+
+
+int
+mdc_inode_xatt_invalidate (xlator_t *this, inode_t *inode)
+{
+        int              ret = -1;
+        struct md_cache *mdc = NULL;
+
+        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
+                goto out;
+
+        LOCK (&mdc->lock);
+        {
+		mdc->xa_time = 0;
+        }
+        UNLOCK (&mdc->lock);
+
+out:
+        return ret;
+}
+
+
 void
 mdc_load_reqs (xlator_t *this, dict_t *dict)
 {
@@ -566,7 +718,7 @@ is_mdc_key_satisfied (const char *key)
 		return 0;
 
 	for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
-		if (!mdc_keys[i].check)
+		if (!mdc_keys[i].load)
 			continue;
 		if (strcmp (mdc_key, key) == 0)
 			return 1;
@@ -576,13 +728,15 @@ is_mdc_key_satisfied (const char *key)
 }
 
 
-static void
+static int
 checkfn (dict_t *this, char *key, data_t *value, void *data)
 {
         struct checkpair *pair = data;
 
 	if (!is_mdc_key_satisfied (key))
 		pair->ret = 0;
+
+        return 0;
 }
 
 
@@ -632,12 +786,13 @@ out:
 
 int
 mdc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            dict_t *xattr_req)
+            dict_t *xdata)
 {
         int          ret = 0;
         struct iatt  stbuf = {0, };
         struct iatt  postparent = {0, };
         dict_t      *xattr_rsp = NULL;
+        dict_t      *xattr_alloc = NULL;
         mdc_local_t *local = NULL;
 
 
@@ -645,18 +800,25 @@ mdc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         if (!local)
                 goto uncached;
 
+	if (!loc->name)
+		/* A nameless discovery is dangerous to cache. We
+		   perform nameless lookup with the intention of
+		   re-establishing an inode "properly"
+		*/
+		goto uncached;
+
         loc_copy (&local->loc, loc);
 
         ret = mdc_inode_iatt_get (this, loc->inode, &stbuf);
         if (ret != 0)
                 goto uncached;
 
-        if (xattr_req) {
+        if (xdata) {
                 ret = mdc_inode_xatt_get (this, loc->inode, &xattr_rsp);
                 if (ret != 0)
                         goto uncached;
 
-                if (!mdc_xattr_satisfied (this, xattr_req, xattr_rsp))
+                if (!mdc_xattr_satisfied (this, xdata, xattr_rsp))
                         goto uncached;
         }
 
@@ -669,22 +831,25 @@ mdc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 
 uncached:
-	if (xattr_req)
-		mdc_load_reqs (this, xattr_req);
+	if (!xdata)
+		xdata = xattr_alloc = dict_new ();
+	if (xdata)
+		mdc_load_reqs (this, xdata);
 
         STACK_WIND (frame, mdc_lookup_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lookup, loc, xattr_req);
+                    FIRST_CHILD (this)->fops->lookup, loc, xdata);
 
         if (xattr_rsp)
                 dict_unref (xattr_rsp);
-
+	if (xattr_alloc)
+		dict_unref (xattr_alloc);
         return 0;
 }
 
 
 int
 mdc_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iatt *buf)
+              int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -698,14 +863,14 @@ mdc_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         mdc_inode_iatt_set (this, local->loc.inode, buf);
 
 out:
-        MDC_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
+        MDC_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
 
         return 0;
 }
 
 
 int
-mdc_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+mdc_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         int           ret;
         struct iatt   stbuf;
@@ -721,21 +886,22 @@ mdc_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         if (ret != 0)
                 goto uncached;
 
-        MDC_STACK_UNWIND (stat, frame, 0, 0, &stbuf);
+        MDC_STACK_UNWIND (stat, frame, 0, 0, &stbuf, xdata);
 
         return 0;
 
 uncached:
         STACK_WIND (frame, mdc_stat_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
-                    loc);
+                    loc, xdata);
         return 0;
 }
 
 
 int
 mdc_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf)
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -749,14 +915,14 @@ mdc_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         mdc_inode_iatt_set (this, local->fd->inode, buf);
 
 out:
-        MDC_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
+        MDC_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
 
         return 0;
 }
 
 
 int
-mdc_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+mdc_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         int           ret;
         struct iatt   stbuf;
@@ -772,14 +938,14 @@ mdc_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
         if (ret != 0)
                 goto uncached;
 
-        MDC_STACK_UNWIND (fstat, frame, 0, 0, &stbuf);
+        MDC_STACK_UNWIND (fstat, frame, 0, 0, &stbuf, xdata);
 
         return 0;
 
 uncached:
         STACK_WIND (frame, mdc_fstat_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
@@ -787,7 +953,7 @@ uncached:
 int
 mdc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf)
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -799,10 +965,11 @@ mdc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!local)
                 goto out;
 
-        mdc_inode_iatt_set (this, local->loc.inode, postbuf);
+        mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf);
 
 out:
-        MDC_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf, postbuf);
+        MDC_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
@@ -810,7 +977,7 @@ out:
 
 int
 mdc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              off_t offset)
+              off_t offset, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -820,7 +987,7 @@ mdc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         STACK_WIND (frame, mdc_truncate_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
-                    loc, offset);
+                    loc, offset, xdata);
         return 0;
 }
 
@@ -828,7 +995,7 @@ mdc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
 int
 mdc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
-                   struct iatt *prebuf, struct iatt *postbuf)
+                   struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -840,10 +1007,11 @@ mdc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!local)
                 goto out;
 
-        mdc_inode_iatt_set (this, local->fd->inode, postbuf);
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
 
 out:
-        MDC_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf);
+        MDC_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
@@ -851,7 +1019,7 @@ out:
 
 int
 mdc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               off_t offset)
+               off_t offset, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -861,7 +1029,7 @@ mdc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         STACK_WIND (frame, mdc_ftruncate_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset);
+                    fd, offset, xdata);
         return 0;
 }
 
@@ -869,7 +1037,8 @@ mdc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int
 mdc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent, struct iatt *postparent)
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
         mdc_local_t *local = NULL;
 
@@ -891,25 +1060,25 @@ mdc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 out:
         MDC_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                          preparent, postparent);
+                          preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 mdc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-           mode_t mode, dev_t rdev, dict_t *params)
+           mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
         local = mdc_local_get (frame);
 
         loc_copy (&local->loc, loc);
-        local->xattr = dict_ref (params);
+        local->xattr = dict_ref (xdata);
 
         STACK_WIND (frame, mdc_mknod_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, params);
+                    loc, mode, rdev, umask, xdata);
         return 0;
 }
 
@@ -917,7 +1086,8 @@ mdc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
 int
 mdc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent, struct iatt *postparent)
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
         mdc_local_t *local = NULL;
 
@@ -939,25 +1109,25 @@ mdc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 out:
         MDC_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
-                          preparent, postparent);
+                          preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 mdc_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-           mode_t mode, dict_t *params)
+           mode_t mode, mode_t umask, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
         local = mdc_local_get (frame);
 
         loc_copy (&local->loc, loc);
-        local->xattr = dict_ref (params);
+        local->xattr = dict_ref (xdata);
 
         STACK_WIND (frame, mdc_mkdir_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, xdata);
         return 0;
 }
 
@@ -965,7 +1135,7 @@ mdc_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
 int
 mdc_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
-                struct iatt *preparent, struct iatt *postparent)
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         mdc_local_t *local = NULL;
 
@@ -987,13 +1157,14 @@ mdc_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         MDC_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                          preparent, postparent);
+                          preparent, postparent, xdata);
         return 0;
 }
 
 
 int
-mdc_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+mdc_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+            dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1003,7 +1174,7 @@ mdc_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
         STACK_WIND (frame, mdc_unlink_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    loc);
+                    loc, xflag, xdata);
         return 0;
 }
 
@@ -1011,7 +1182,7 @@ mdc_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 int
 mdc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
-                struct iatt *preparent, struct iatt *postparent)
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         mdc_local_t *local = NULL;
 
@@ -1029,13 +1200,14 @@ mdc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         MDC_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                          preparent, postparent);
+                          preparent, postparent, xdata);
         return 0;
 }
 
 
 int
-mdc_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag)
+mdc_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+           dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1045,7 +1217,7 @@ mdc_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag)
 
         STACK_WIND (frame, mdc_rmdir_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
-                    loc, flag);
+                    loc, flag, xdata);
         return 0;
 }
 
@@ -1053,7 +1225,8 @@ mdc_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag)
 int
 mdc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
         mdc_local_t *local = NULL;
 
@@ -1074,14 +1247,14 @@ mdc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 out:
         MDC_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                          preparent, postparent);
+                          preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 mdc_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-             loc_t *loc, dict_t *params)
+             loc_t *loc, mode_t umask, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1093,7 +1266,7 @@ mdc_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
 
         STACK_WIND (frame, mdc_symlink_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                    linkname, loc, params);
+                    linkname, loc, umask, xdata);
         return 0;
 }
 
@@ -1102,7 +1275,8 @@ int
 mdc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
                 struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent)
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
 {
         mdc_local_t *local = NULL;
 
@@ -1131,14 +1305,15 @@ mdc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 out:
         MDC_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
-                          preoldparent, postoldparent, prenewparent, postnewparent);
+                          preoldparent, postoldparent, prenewparent,
+                          postnewparent, xdata);
         return 0;
 }
 
 
 int
 mdc_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc)
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1149,7 +1324,7 @@ mdc_rename (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, mdc_rename_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
         return 0;
 }
 
@@ -1157,7 +1332,7 @@ mdc_rename (call_frame_t *frame, xlator_t *this,
 int
 mdc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int32_t op_ret, int32_t op_errno, inode_t *inode, struct iatt *buf,
-              struct iatt *preparent, struct iatt *postparent)
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         mdc_local_t *local = NULL;
 
@@ -1178,14 +1353,14 @@ mdc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 out:
         MDC_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                          preparent, postparent);
+                          preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 mdc_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc)
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1196,7 +1371,7 @@ mdc_link (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, mdc_link_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
         return 0;
 }
 
@@ -1204,7 +1379,8 @@ mdc_link (call_frame_t *frame, xlator_t *this,
 int
 mdc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent, struct iatt *postparent)
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
         mdc_local_t *local = NULL;
 
@@ -1226,25 +1402,25 @@ mdc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 out:
         MDC_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                          preparent, postparent);
+                          preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 mdc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-            mode_t mode, fd_t *fd, dict_t *params)
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
         local = mdc_local_get (frame);
 
         loc_copy (&local->loc, loc);
-        local->xattr = dict_ref (params);
+        local->xattr = dict_ref (xdata);
 
         STACK_WIND (frame, mdc_create_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
@@ -1253,7 +1429,7 @@ int
 mdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno,
                struct iovec *vector, int32_t count,
-               struct iatt *stbuf, struct iobref *iobref)
+               struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1269,7 +1445,7 @@ mdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 out:
         MDC_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                          stbuf, iobref);
+                          stbuf, iobref, xdata);
 
         return 0;
 }
@@ -1277,7 +1453,7 @@ out:
 
 int
 mdc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-           off_t offset, uint32_t flags)
+           off_t offset, uint32_t flags, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1287,7 +1463,7 @@ mdc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
         STACK_WIND (frame, mdc_readv_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
@@ -1295,7 +1471,7 @@ mdc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 int
 mdc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno,
-               struct iatt *prebuf, struct iatt *postbuf)
+               struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1307,10 +1483,11 @@ mdc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!local)
                 goto out;
 
-        mdc_inode_iatt_set (this, local->fd->inode, postbuf);
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
 
 out:
-        MDC_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        MDC_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
@@ -1318,7 +1495,8 @@ out:
 
 int
 mdc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-            int count, off_t offset, uint32_t flags, struct iobref *iobref)
+            int count, off_t offset, uint32_t flags, struct iobref *iobref,
+            dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1328,7 +1506,7 @@ mdc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
 
         STACK_WIND (frame, mdc_writev_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
         return 0;
 }
 
@@ -1336,7 +1514,7 @@ mdc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
 int
 mdc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *prebuf, struct iatt *postbuf)
+                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1350,10 +1528,11 @@ mdc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!local)
                 goto out;
 
-        mdc_inode_iatt_set (this, local->loc.inode, postbuf);
+	mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf);
 
 out:
-        MDC_STACK_UNWIND (setattr, frame, op_ret, op_errno, prebuf, postbuf);
+        MDC_STACK_UNWIND (setattr, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
@@ -1361,7 +1540,7 @@ out:
 
 int
 mdc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int valid)
+             struct iatt *stbuf, int valid, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1371,7 +1550,7 @@ mdc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         STACK_WIND (frame, mdc_setattr_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid);
+                    loc, stbuf, valid, xdata);
         return 0;
 }
 
@@ -1379,7 +1558,7 @@ mdc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 int
 mdc_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf)
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1391,10 +1570,11 @@ mdc_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!local)
                 goto out;
 
-        mdc_inode_iatt_set (this, local->fd->inode, postbuf);
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
 
 out:
-        MDC_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, prebuf, postbuf);
+        MDC_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
@@ -1402,7 +1582,7 @@ out:
 
 int
 mdc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iatt *stbuf, int valid)
+              struct iatt *stbuf, int valid, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1410,9 +1590,9 @@ mdc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         local->fd = fd_ref (fd);
 
-        STACK_WIND (frame, mdc_setattr_cbk,
+        STACK_WIND (frame, mdc_fsetattr_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid);
+                    fd, stbuf, valid, xdata);
         return 0;
 }
 
@@ -1420,7 +1600,7 @@ mdc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int
 mdc_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno,
-               struct iatt *prebuf, struct iatt *postbuf)
+               struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1432,17 +1612,19 @@ mdc_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!local)
                 goto out;
 
-        mdc_inode_iatt_set (this, local->fd->inode, postbuf);
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
 
 out:
-        MDC_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        MDC_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
 
         return 0;
 }
 
 
 int
-mdc_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+mdc_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+           dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1452,14 +1634,14 @@ mdc_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
 
         STACK_WIND (frame, mdc_fsync_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
-                    fd, datasync);
+                    fd, datasync, xdata);
         return 0;
 }
 
 
 int
 mdc_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1473,8 +1655,10 @@ mdc_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         mdc_inode_xatt_update (this, local->loc.inode, local->xattr);
 
+	mdc_inode_iatt_invalidate (this, local->loc.inode);
+
 out:
-        MDC_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        MDC_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -1482,7 +1666,7 @@ out:
 
 int
 mdc_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xattr, int flags)
+              dict_t *xattr, int flags, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1493,14 +1677,14 @@ mdc_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         STACK_WIND (frame, mdc_setxattr_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
-                    loc, xattr, flags);
+                    loc, xattr, flags, xdata);
         return 0;
 }
 
 
 int
 mdc_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
+		   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1514,8 +1698,9 @@ mdc_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         mdc_inode_xatt_update (this, local->fd->inode, local->xattr);
 
+	mdc_inode_iatt_invalidate (this, local->fd->inode);
 out:
-        MDC_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
+        MDC_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -1523,7 +1708,7 @@ out:
 
 int
 mdc_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	       dict_t *xattr, int flags)
+	       dict_t *xattr, int flags, dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1534,13 +1719,14 @@ mdc_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         STACK_WIND (frame, mdc_fsetxattr_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, xattr, flags);
+                    fd, xattr, flags, xdata);
         return 0;
 }
 
 int
 mdc_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xattr)
+		  int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                  dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1554,16 +1740,18 @@ mdc_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         mdc_inode_xatt_update (this, local->loc.inode, xattr);
 
 out:
-        MDC_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+        MDC_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
         return 0;
 }
 
 
 int
-mdc_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key)
+mdc_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
+              dict_t *xdata)
 {
         int           ret;
+	int           op_errno = ENODATA;
         mdc_local_t  *local = NULL;
 	dict_t       *xattr = NULL;
 
@@ -1580,24 +1768,27 @@ mdc_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key)
 	if (ret != 0)
 		goto uncached;
 
-	if (!dict_get (xattr, (char *)key))
-		goto uncached;
+	if (!xattr || !dict_get (xattr, (char *)key)) {
+		ret = -1;
+		op_errno = ENODATA;
+	}
 
-        MDC_STACK_UNWIND (getxattr, frame, 0, 0, xattr);
+        MDC_STACK_UNWIND (getxattr, frame, ret, op_errno, xattr, xdata);
 
         return 0;
 
 uncached:
         STACK_WIND (frame, mdc_getxattr_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
-                    loc, key);
+                    loc, key, xdata);
         return 0;
 }
 
 
 int
 mdc_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, dict_t *xattr)
+		   int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                   dict_t *xdata)
 {
         mdc_local_t  *local = NULL;
 
@@ -1611,18 +1802,20 @@ mdc_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         mdc_inode_xatt_update (this, local->fd->inode, xattr);
 
 out:
-        MDC_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr);
+        MDC_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr, xdata);
 
         return 0;
 }
 
 
 int
-mdc_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key)
+mdc_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+               dict_t *xdata)
 {
         int           ret;
         mdc_local_t  *local = NULL;
 	dict_t       *xattr = NULL;
+	int           op_errno = ENODATA;
 
         local = mdc_local_get (frame);
         if (!local)
@@ -1637,24 +1830,117 @@ mdc_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key)
 	if (ret != 0)
 		goto uncached;
 
-	if (!dict_get (xattr, (char *)key))
-		goto uncached;
+	if (!xattr || !dict_get (xattr, (char *)key)) {
+		ret = -1;
+		op_errno = ENODATA;
+	}
 
-        MDC_STACK_UNWIND (fgetxattr, frame, 0, 0, xattr);
+        MDC_STACK_UNWIND (fgetxattr, frame, ret, op_errno, xattr, xdata);
 
         return 0;
 
 uncached:
         STACK_WIND (frame, mdc_fgetxattr_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, key);
+                    fd, key, xdata);
+        return 0;
+}
+
+int
+mdc_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+	if (local->key)
+		mdc_inode_xatt_unset (this, local->loc.inode, local->key);
+	else
+		mdc_inode_xatt_invalidate (this, local->loc.inode);
+
+	mdc_inode_iatt_invalidate (this, local->loc.inode);
+out:
+        MDC_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		 const char *name, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+	loc_copy (&local->loc, loc);
+
+	local->key = gf_strdup (name);
+
+        STACK_WIND (frame, mdc_removexattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+
+int
+mdc_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+	if (local->key)
+		mdc_inode_xatt_unset (this, local->fd->inode, local->key);
+	else
+		mdc_inode_xatt_invalidate (this, local->fd->inode);
+
+	mdc_inode_iatt_invalidate (this, local->fd->inode);
+out:
+        MDC_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+
+int
+mdc_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  const char *name, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = mdc_local_get (frame);
+
+	local->fd = fd_ref (fd);
+
+	local->key = gf_strdup (name);
+
+        STACK_WIND (frame, mdc_fremovexattr_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
         return 0;
 }
 
 
 int
 mdc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, gf_dirent_t *entries)
+		  int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
         gf_dirent_t *entry      = NULL;
 
@@ -1669,42 +1955,187 @@ mdc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
 unwind:
-	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
 
 int
 mdc_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	      size_t size, off_t offset, dict_t *xattr_req)
+	      size_t size, off_t offset, dict_t *xdata)
 {
+	dict_t *xattr_alloc = NULL;
+
+	if (!xdata)
+		xdata = xattr_alloc = dict_new ();
+	if (xdata)
+		mdc_load_reqs (this, xdata);
+
 	STACK_WIND (frame, mdc_readdirp_cbk,
 		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readdirp,
-		    fd, size, offset, xattr_req);
+		    fd, size, offset, xdata);
+	if (xattr_alloc)
+		dict_unref (xattr_alloc);
 	return 0;
 }
 
+int
+mdc_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+		int op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
 
 int
 mdc_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	     size_t size, off_t offset)
+	     size_t size, off_t offset, dict_t *xdata)
 {
-	dict_t *xattr_req = NULL;
-
-	xattr_req = dict_new ();
+        int need_unref = 0;
+	struct mdc_conf *conf = this->private;
 
-	if (xattr_req) {
-		mdc_load_reqs (this, xattr_req);
+	if (!conf->force_readdirp) {
+		STACK_WIND(frame, mdc_readdir_cbk, FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->readdir, fd, size, offset,
+			   xdata);
+		return 0;
 	}
 
-	STACK_WIND (frame, mdc_readdirp_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readdirp,
-		    fd, size, offset, xattr_req);
+	if (!xdata) {
+                xdata = dict_new ();
+                need_unref = 1;
+        }
+
+        if (xdata)
+		mdc_load_reqs (this, xdata);
+
+	STACK_WIND(frame, mdc_readdirp_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, size, offset,
+		   xdata);
+
+        if (need_unref && xdata)
+                dict_unref (xdata);
+
+	return 0;
+}
+
+int
+mdc_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+int mdc_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		  off_t offset, size_t len, dict_t *xdata)
+{
+	mdc_local_t *local;
+
+	local = mdc_local_get(frame);
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame, mdc_fallocate_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		   xdata);
+
+	return 0;
+}
+
+int
+mdc_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, postbuf,
+                         xdata);
+
+        return 0;
+}
+
+int mdc_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+		size_t len, dict_t *xdata)
+{
+	mdc_local_t *local;
+
+	local = mdc_local_get(frame);
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame, mdc_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len,
+		   xdata);
 
-        dict_unref (xattr_req);
 	return 0;
 }
 
+int
+mdc_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        mdc_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret != 0)
+                goto out;
+
+        if (!local)
+                goto out;
+
+        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+
+out:
+        MDC_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                         xdata);
+
+        return 0;
+}
+
+int mdc_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                off_t len, dict_t *xdata)
+{
+        mdc_local_t *local;
+
+        local = mdc_local_get(frame);
+        local->fd = fd_ref(fd);
+
+        STACK_WIND(frame, mdc_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len,
+                   xdata);
+
+        return 0;
+}
+
 
 int
 mdc_forget (xlator_t *this, inode_t *inode)
@@ -1716,13 +2147,51 @@ mdc_forget (xlator_t *this, inode_t *inode)
 
 
 int
+is_strpfx (const char *str1, const char *str2)
+{
+	/* is one of the string a prefix of the other? */
+	int i;
+
+	for (i = 0; str1[i] == str2[i]; i++) {
+		if (!str1[i] || !str2[i])
+			break;
+	}
+
+	return !(str1[i] && str2[i]);
+}
+
+
+int
+mdc_key_load_set (struct mdc_key *keys, char *pattern, gf_boolean_t val)
+{
+	struct mdc_key *key = NULL;
+
+	for (key = keys; key->name; key++) {
+		if (is_strpfx (key->name, pattern))
+			key->load = val;
+	}
+
+	return 0;
+}
+
+
+int
 reconfigure (xlator_t *this, dict_t *options)
 {
 	struct mdc_conf *conf = NULL;
 
 	conf = this->private;
 
-	GF_OPTION_RECONF ("timeout", conf->timeout, options, int32, out);
+	GF_OPTION_RECONF ("md-cache-timeout", conf->timeout, options, int32, out);
+
+	GF_OPTION_RECONF ("cache-selinux", conf->cache_selinux, options, bool, out);
+	mdc_key_load_set (mdc_keys, "security.", conf->cache_selinux);
+
+	GF_OPTION_RECONF ("cache-posix-acl", conf->cache_posix_acl, options, bool, out);
+	mdc_key_load_set (mdc_keys, "system.posix_acl_", conf->cache_posix_acl);
+
+	GF_OPTION_RECONF("force-readdirp", conf->force_readdirp, options, bool, out);
+
 out:
 	return 0;
 }
@@ -1748,8 +2217,15 @@ init (xlator_t *this)
 		return -1;
 	}
 
-        GF_OPTION_INIT ("timeout", conf->timeout, int32, out);
+        GF_OPTION_INIT ("md-cache-timeout", conf->timeout, int32, out);
+
+	GF_OPTION_INIT ("cache-selinux", conf->cache_selinux, bool, out);
+	mdc_key_load_set (mdc_keys, "security.", conf->cache_selinux);
 
+	GF_OPTION_INIT ("cache-posix-acl", conf->cache_posix_acl, bool, out);
+	mdc_key_load_set (mdc_keys, "system.posix_acl_", conf->cache_posix_acl);
+
+	GF_OPTION_INIT("force-readdirp", conf->force_readdirp, bool, out);
 out:
 	this->private = conf;
 
@@ -1787,8 +2263,13 @@ struct xlator_fops fops = {
         .fsetxattr   = mdc_fsetxattr,
         .getxattr    = mdc_getxattr,
         .fgetxattr   = mdc_fgetxattr,
+	.removexattr = mdc_removexattr,
+	.fremovexattr= mdc_fremovexattr,
 	.readdirp    = mdc_readdirp,
-	.readdir     = mdc_readdir
+	.readdir     = mdc_readdir,
+	.fallocate   = mdc_fallocate,
+	.discard     = mdc_discard,
+        .zerofill    = mdc_zerofill,
 };
 
 
@@ -1797,11 +2278,26 @@ struct xlator_cbks cbks = {
 };
 
 struct volume_options options[] = {
-        { .key = {"timeout"},
+	{ .key = {"cache-selinux"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "false",
+	},
+	{ .key = {"cache-posix-acl"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "false",
+	},
+        { .key = {"md-cache-timeout"},
           .type = GF_OPTION_TYPE_INT,
           .min = 0,
           .max = 60,
           .default_value = "1",
           .description = "Time period after which cache has to be refreshed",
         },
+	{ .key = {"force-readdirp"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "true",
+	  .description = "Convert all readdir requests to readdirplus to "
+			 "collect stat info on each entry.",
+	},
+    { .key = {NULL} },
 };
diff --git a/xlators/performance/open-behind/Makefile.am b/xlators/performance/open-behind/Makefile.am
new file mode 100644
index 000000000..af437a64d
--- /dev/null
+++ b/xlators/performance/open-behind/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src
diff --git a/xlators/performance/open-behind/src/Makefile.am b/xlators/performance/open-behind/src/Makefile.am
new file mode 100644
index 000000000..125285707
--- /dev/null
+++ b/xlators/performance/open-behind/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = open-behind.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
+
+open_behind_la_LDFLAGS = -module -avoid-version
+
+open_behind_la_SOURCES = open-behind.c
+open_behind_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = open-behind-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/performance/open-behind/src/open-behind-mem-types.h b/xlators/performance/open-behind/src/open-behind-mem-types.h
new file mode 100644
index 000000000..1e94296f4
--- /dev/null
+++ b/xlators/performance/open-behind/src/open-behind-mem-types.h
@@ -0,0 +1,21 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __OB_MEM_TYPES_H__
+#define __OB_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_ob_mem_types_ {
+        gf_ob_mt_fd_t   = gf_common_mt_end + 1,
+	gf_ob_mt_conf_t,
+        gf_ob_mt_end
+};
+#endif
diff --git a/xlators/performance/open-behind/src/open-behind.c b/xlators/performance/open-behind/src/open-behind.c
new file mode 100644
index 000000000..742e4df3f
--- /dev/null
+++ b/xlators/performance/open-behind/src/open-behind.c
@@ -0,0 +1,1020 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "open-behind-mem-types.h"
+#include "xlator.h"
+#include "statedump.h"
+#include "call-stub.h"
+#include "defaults.h"
+
+typedef struct ob_conf {
+	gf_boolean_t  use_anonymous_fd; /* use anonymous FDs wherever safe
+					   e.g - fstat() readv()
+
+					   whereas for fops like writev(), lk(),
+					   the fd is important for side effects
+					   like mandatory locks
+					*/
+	gf_boolean_t  lazy_open; /* delay backend open as much as possible */
+        gf_boolean_t  read_after_open; /* instead of sending readvs on
+                                               anonymous fds, open the file
+                                               first and then send readv i.e
+                                               similar to what writev does
+                                            */
+} ob_conf_t;
+
+
+typedef struct ob_fd {
+	call_frame_t     *open_frame;
+	loc_t             loc;
+	dict_t           *xdata;
+	int               flags;
+	int               op_errno;
+	struct list_head  list;
+} ob_fd_t;
+
+
+ob_fd_t *
+__ob_fd_ctx_get (xlator_t *this, fd_t *fd)
+{
+	uint64_t    value = 0;
+	int         ret = -1;
+	ob_fd_t    *ob_fd = NULL;
+
+	ret = __fd_ctx_get (fd, this, &value);
+	if (ret)
+		return NULL;
+
+	ob_fd = (void *) ((long) value);
+
+	return ob_fd;
+}
+
+
+ob_fd_t *
+ob_fd_ctx_get (xlator_t *this, fd_t *fd)
+{
+	ob_fd_t  *ob_fd = NULL;
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+	}
+	UNLOCK (&fd->lock);
+
+	return ob_fd;
+}
+
+
+int
+__ob_fd_ctx_set (xlator_t *this, fd_t *fd, ob_fd_t *ob_fd)
+{
+	uint64_t    value = 0;
+	int         ret = -1;
+
+	value = (long) ((void *) ob_fd);
+
+	ret = __fd_ctx_set (fd, this, value);
+
+	return ret;
+}
+
+
+int
+ob_fd_ctx_set (xlator_t *this, fd_t *fd, ob_fd_t *ob_fd)
+{
+	int ret = -1;
+
+	LOCK (&fd->lock);
+	{
+		ret = __ob_fd_ctx_set (this, fd, ob_fd);
+	}
+	UNLOCK (&fd->lock);
+
+	return ret;
+}
+
+
+ob_fd_t *
+ob_fd_new (void)
+{
+	ob_fd_t  *ob_fd = NULL;
+
+	ob_fd = GF_CALLOC (1, sizeof (*ob_fd), gf_ob_mt_fd_t);
+
+	INIT_LIST_HEAD (&ob_fd->list);
+
+	return ob_fd;
+}
+
+
+void
+ob_fd_free (ob_fd_t *ob_fd)
+{
+	loc_wipe (&ob_fd->loc);
+
+	if (ob_fd->xdata)
+		dict_unref (ob_fd->xdata);
+
+	if (ob_fd->open_frame)
+		STACK_DESTROY (ob_fd->open_frame->root);
+
+	GF_FREE (ob_fd);
+}
+
+
+int
+ob_wake_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	     int op_ret, int op_errno, fd_t *fd_ret, dict_t *xdata)
+{
+	fd_t              *fd = NULL;
+	struct list_head   list;
+	ob_fd_t           *ob_fd = NULL;
+	call_stub_t       *stub = NULL, *tmp = NULL;
+
+	fd = frame->local;
+	frame->local = NULL;
+
+	INIT_LIST_HEAD (&list);
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+
+		list_splice_init (&ob_fd->list, &list);
+
+		if (op_ret < 0) {
+			/* mark fd BAD for ever */
+			ob_fd->op_errno = op_errno;
+		} else {
+			__fd_ctx_del (fd, this, NULL);
+			ob_fd_free (ob_fd);
+		}
+	}
+	UNLOCK (&fd->lock);
+
+	list_for_each_entry_safe (stub, tmp, &list, list) {
+		list_del_init (&stub->list);
+
+		if (op_ret < 0)
+			call_unwind_error (stub, -1, op_errno);
+		else
+			call_resume (stub);
+	}
+
+	fd_unref (fd);
+
+	STACK_DESTROY (frame->root);
+
+	return 0;
+}
+
+
+int
+ob_fd_wake (xlator_t *this, fd_t *fd)
+{
+	call_frame_t *frame = NULL;
+	ob_fd_t      *ob_fd = NULL;
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+		if (!ob_fd)
+			goto unlock;
+
+		frame = ob_fd->open_frame;
+		ob_fd->open_frame = NULL;
+	}
+unlock:
+	UNLOCK (&fd->lock);
+
+	if (frame) {
+		frame->local = fd_ref (fd);
+
+		STACK_WIND (frame, ob_wake_cbk, FIRST_CHILD (this),
+			    FIRST_CHILD (this)->fops->open,
+			    &ob_fd->loc, ob_fd->flags, fd, ob_fd->xdata);
+	}
+
+	return 0;
+}
+
+
+int
+open_and_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
+{
+	ob_fd_t  *ob_fd = NULL;
+	int       op_errno = 0;
+
+	if (!fd)
+		goto nofd;
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+		if (!ob_fd)
+			goto unlock;
+
+		if (ob_fd->op_errno) {
+			op_errno = ob_fd->op_errno;
+			goto unlock;
+		}
+
+		list_add_tail (&stub->list, &ob_fd->list);
+	}
+unlock:
+	UNLOCK (&fd->lock);
+
+nofd:
+	if (op_errno)
+		call_unwind_error (stub, -1, op_errno);
+	else if (ob_fd)
+		ob_fd_wake (this, fd);
+	else
+		call_resume (stub);
+
+	return 0;
+}
+
+
+int
+ob_open_behind (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+		fd_t *fd, dict_t *xdata)
+{
+	ob_fd_t    *ob_fd = NULL;
+	int         ret = -1;
+	ob_conf_t  *conf = NULL;
+
+
+	conf = this->private;
+
+	if (flags & O_TRUNC) {
+		STACK_WIND (frame, default_open_cbk,
+			    FIRST_CHILD (this), FIRST_CHILD (this)->fops->open,
+			    loc, flags, fd, xdata);
+		return 0;
+	}
+
+	ob_fd = ob_fd_new ();
+	if (!ob_fd)
+		goto enomem;
+
+	ob_fd->open_frame = copy_frame (frame);
+	if (!ob_fd->open_frame)
+		goto enomem;
+	ret = loc_copy (&ob_fd->loc, loc);
+	if (ret)
+		goto enomem;
+
+	ob_fd->flags = flags;
+	if (xdata)
+		ob_fd->xdata = dict_ref (xdata);
+
+	ret = ob_fd_ctx_set (this, fd, ob_fd);
+	if (ret)
+		goto enomem;
+
+	fd_ref (fd);
+
+	STACK_UNWIND_STRICT (open, frame, 0, 0, fd, xdata);
+
+	if (!conf->lazy_open)
+		ob_fd_wake (this, fd);
+
+	fd_unref (fd);
+
+	return 0;
+enomem:
+	if (ob_fd) {
+		if (ob_fd->open_frame)
+			STACK_DESTROY (ob_fd->open_frame->root);
+		loc_wipe (&ob_fd->loc);
+		if (ob_fd->xdata)
+			dict_unref (ob_fd->xdata);
+		GF_FREE (ob_fd);
+	}
+
+	return -1;
+}
+
+
+int
+ob_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	 fd_t *fd, dict_t *xdata)
+{
+	fd_t         *old_fd = NULL;
+	int           ret = -1;
+	int           op_errno = 0;
+	call_stub_t  *stub = NULL;
+
+	old_fd = fd_lookup (fd->inode, 0);
+	if (old_fd) {
+		/* open-behind only when this is the first FD */
+		stub = fop_open_stub (frame, default_open_resume,
+				      loc, flags, fd, xdata);
+		if (!stub) {
+			op_errno = ENOMEM;
+			fd_unref (old_fd);
+			goto err;
+		}
+
+		open_and_resume (this, old_fd, stub);
+
+		fd_unref (old_fd);
+
+		return 0;
+	}
+
+	ret = ob_open_behind (frame, this, loc, flags, fd, xdata);
+	if (ret) {
+		op_errno = ENOMEM;
+		goto err;
+	}
+
+	return 0;
+err:
+	gf_log (this->name, GF_LOG_ERROR, "%s: %s", loc->path,
+		strerror (op_errno));
+
+	STACK_UNWIND_STRICT (open, frame, -1, op_errno, 0, 0);
+
+	return 0;
+}
+
+
+fd_t *
+ob_get_wind_fd (xlator_t *this, fd_t *fd)
+{
+	ob_conf_t  *conf = NULL;
+	ob_fd_t    *ob_fd = NULL;
+
+	conf = this->private;
+
+	ob_fd = ob_fd_ctx_get (this, fd);
+
+	if (ob_fd && conf->use_anonymous_fd)
+		return fd_anonymous (fd->inode);
+
+	return fd_ref (fd);
+}
+
+
+int
+ob_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	  off_t offset, uint32_t flags, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+	fd_t         *wind_fd = NULL;
+        ob_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->read_after_open)
+                wind_fd = ob_get_wind_fd (this, fd);
+        else
+                wind_fd = fd_ref (fd);
+
+	stub = fop_readv_stub (frame, default_readv_resume, wind_fd,
+			       size, offset, flags, xdata);
+	fd_unref (wind_fd);
+
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, wind_fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM, 0, 0, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
+	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
+	   dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_writev_stub (frame, default_writev_resume, fd, iov, count,
+				offset, flags, iobref, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+	fd_t         *wind_fd = NULL;
+
+	wind_fd = ob_get_wind_fd (this, fd);
+
+	stub = fop_fstat_stub (frame, default_fstat_resume, wind_fd, xdata);
+
+	fd_unref (wind_fd);
+
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, wind_fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	call_stub_t   *stub = NULL;
+	ob_fd_t       *ob_fd = NULL;
+	gf_boolean_t   unwind = _gf_false;
+
+	LOCK (&fd->lock);
+	{
+		ob_fd = __ob_fd_ctx_get (this, fd);
+		if (ob_fd && ob_fd->open_frame)
+			/* if open() was never wound to backend,
+			   no need to wind flush() either.
+			*/
+			unwind = _gf_true;
+	}
+	UNLOCK (&fd->lock);
+
+	if (unwind)
+		goto unwind;
+
+	stub = fop_flush_stub (frame, default_flush_resume, fd, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, 0);
+
+	return 0;
+
+unwind:
+	STACK_UNWIND_STRICT (flush, frame, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int flag,
+	  dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fsync_stub (frame, default_fsync_resume, fd, flag, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+       struct gf_flock *flock, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, flock, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, 0, 0);
+
+	return 0;
+}
+
+int
+ob_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	      dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset,
+				   xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+	      int flags, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume, fd, xattr,
+				   flags, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, 0);
+
+	return 0;
+}
+
+
+int
+ob_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+	      dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name,
+				   xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		 const char *name, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fremovexattr_stub (frame, default_fremovexattr_resume, fd,
+				      name, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOMEM, 0);
+
+	return 0;
+}
+
+
+int
+ob_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+	     int cmd, struct gf_flock *flock, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_finodelk_stub (frame, default_finodelk_resume, volume, fd,
+				  cmd, flock, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, 0);
+
+	return 0;
+}
+
+
+int
+ob_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+	     const char *basename, entrylk_cmd cmd, entrylk_type type,
+	     dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fentrylk_stub (frame, default_fentrylk_resume, volume, fd,
+				  basename, cmd, type, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, 0);
+
+	return 0;
+}
+
+
+int
+ob_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fxattrop_stub (frame, default_fxattrop_resume, fd, optype,
+				  xattr, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     struct iatt *iatt, int valid, dict_t *xdata)
+{
+	call_stub_t  *stub = NULL;
+
+	stub = fop_fsetattr_stub (frame, default_fsetattr_resume, fd,
+				  iatt, valid, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume (this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+int
+ob_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	     off_t offset, size_t len, dict_t *xdata)
+{
+	call_stub_t *stub;
+
+	stub = fop_fallocate_stub(frame, default_fallocate_resume, fd, mode,
+				  offset, len, xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume(this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+	return 0;
+}
+
+int
+ob_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	   size_t len, dict_t *xdata)
+{
+	call_stub_t *stub;
+
+	stub = fop_discard_stub(frame, default_discard_resume, fd, offset, len,
+				xdata);
+	if (!stub)
+		goto err;
+
+	open_and_resume(this, fd, stub);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT(discard, frame, -1, ENOMEM, NULL, NULL, NULL);
+	return 0;
+}
+
+int
+ob_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           off_t len, dict_t *xdata)
+{
+        call_stub_t *stub;
+
+        stub = fop_zerofill_stub(frame, default_zerofill_resume, fd,
+                                 offset, len, xdata);
+        if (!stub)
+                goto err;
+
+        open_and_resume(this, fd, stub);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT(zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+ob_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+	   dict_t *xdata)
+{
+	fd_t         *fd = NULL;
+	call_stub_t  *stub = NULL;
+
+	stub = fop_unlink_stub (frame, default_unlink_resume, loc,
+				xflags, xdata);
+	if (!stub)
+		goto err;
+
+	fd = fd_lookup (loc->inode, 0);
+
+	open_and_resume (this, fd, stub);
+        if (fd)
+                fd_unref (fd);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_rename (call_frame_t *frame, xlator_t *this, loc_t *src, loc_t *dst,
+	   dict_t *xdata)
+{
+	fd_t         *fd = NULL;
+	call_stub_t  *stub = NULL;
+
+	stub = fop_rename_stub (frame, default_rename_resume, src, dst, xdata);
+	if (!stub)
+		goto err;
+
+	if (dst->inode)
+		fd = fd_lookup (dst->inode, 0);
+
+	open_and_resume (this, fd, stub);
+        if (fd)
+                fd_unref (fd);
+
+	return 0;
+err:
+	STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM, 0, 0, 0, 0, 0, 0);
+
+	return 0;
+}
+
+
+int
+ob_release (xlator_t *this, fd_t *fd)
+{
+	ob_fd_t *ob_fd = NULL;
+
+	ob_fd = ob_fd_ctx_get (this, fd);
+
+	ob_fd_free (ob_fd);
+
+	return 0;
+}
+
+
+int
+ob_priv_dump (xlator_t *this)
+{
+        ob_conf_t        *conf       = NULL;
+        char              key_prefix[GF_DUMP_MAX_BUF_LEN];
+
+        conf = this->private;
+
+        if (!conf)
+                return -1;
+
+        gf_proc_dump_build_key (key_prefix, "xlator.performance.open-behind",
+                                "priv");
+
+        gf_proc_dump_add_section (key_prefix);
+
+        gf_proc_dump_write ("use_anonymous_fd", "%d", conf->use_anonymous_fd);
+
+        gf_proc_dump_write ("lazy_open", "%d", conf->lazy_open);
+
+        return 0;
+}
+
+
+int
+ob_fdctx_dump (xlator_t *this, fd_t *fd)
+{
+	ob_fd_t   *ob_fd = NULL;
+        char       key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+	int        ret = 0;
+
+	ret = TRY_LOCK (&fd->lock);
+	if (ret)
+		return 0;
+
+	ob_fd = __ob_fd_ctx_get (this, fd);
+	if (!ob_fd) {
+		UNLOCK (&fd->lock);
+		return 0;
+	}
+
+        gf_proc_dump_build_key (key_prefix, "xlator.performance.open-behind",
+                                "file");
+        gf_proc_dump_add_section (key_prefix);
+
+        gf_proc_dump_write ("fd", "%p", fd);
+
+        gf_proc_dump_write ("open_frame", "%p", ob_fd->open_frame);
+
+        gf_proc_dump_write ("open_frame.root.unique", "%p",
+			    ob_fd->open_frame->root->unique);
+
+	gf_proc_dump_write ("loc.path", "%s", ob_fd->loc.path);
+
+	gf_proc_dump_write ("loc.ino", "%s", uuid_utoa (ob_fd->loc.gfid));
+
+        gf_proc_dump_write ("flags", "%p", ob_fd->open_frame);
+
+	UNLOCK (&fd->lock);
+
+	return 0;
+}
+
+
+int
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_ob_mt_end + 1);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting failed");
+
+        return ret;
+}
+
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        ob_conf_t  *conf = NULL;
+	int         ret = -1;
+
+        conf = this->private;
+
+        GF_OPTION_RECONF ("use-anonymous-fd", conf->use_anonymous_fd, options,
+			  bool, out);
+
+        GF_OPTION_RECONF ("lazy-open", conf->lazy_open, options, bool, out);
+        GF_OPTION_RECONF ("read-after-open", conf->read_after_open, options,
+                          bool, out);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+init (xlator_t *this)
+{
+        ob_conf_t    *conf = NULL;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "FATAL: volume (%s) not configured with exactly one "
+                        "child", this->name);
+                return -1;
+        }
+
+        if (!this->parents)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_ob_mt_conf_t);
+        if (!conf)
+                goto err;
+
+        GF_OPTION_INIT ("use-anonymous-fd", conf->use_anonymous_fd, bool, err);
+
+        GF_OPTION_INIT ("lazy-open", conf->lazy_open, bool, err);
+        GF_OPTION_INIT ("read-after-open", conf->read_after_open, bool, err);
+        this->private = conf;
+
+	return 0;
+err:
+	if (conf)
+                GF_FREE (conf);
+
+        return -1;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        ob_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        GF_FREE (conf);
+
+	return;
+}
+
+
+struct xlator_fops fops = {
+        .open        = ob_open,
+        .readv       = ob_readv,
+        .writev      = ob_writev,
+	.flush       = ob_flush,
+	.fsync       = ob_fsync,
+	.fstat       = ob_fstat,
+	.ftruncate   = ob_ftruncate,
+	.fsetxattr   = ob_fsetxattr,
+	.fgetxattr   = ob_fgetxattr,
+	.fremovexattr = ob_fremovexattr,
+	.finodelk    = ob_finodelk,
+	.fentrylk    = ob_fentrylk,
+	.fxattrop    = ob_fxattrop,
+	.fsetattr    = ob_fsetattr,
+	.fallocate   = ob_fallocate,
+	.discard     = ob_discard,
+        .zerofill    = ob_zerofill,
+	.unlink      = ob_unlink,
+	.rename      = ob_rename,
+	.lk          = ob_lk,
+};
+
+struct xlator_cbks cbks = {
+        .release  = ob_release,
+};
+
+struct xlator_dumpops dumpops = {
+	.priv    = ob_priv_dump,
+	.fdctx   = ob_fdctx_dump,
+};
+
+
+struct volume_options options[] = {
+        { .key  = {"use-anonymous-fd"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "yes",
+          .description = "For read operations, use anonymous FD when "
+          "original FD is open-behind and not yet opened in the backend.",
+        },
+        { .key  = {"lazy-open"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "yes",
+          .description = "Perform open in the backend only when a necessary "
+          "FOP arrives (e.g writev on the FD, unlink of the file). When option "
+          "is disabled, perform backend open right after unwinding open().",
+        },
+        { .key  = {"read-after-open"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+          .description = "read is sent only after actual open happens and real "
+          "fd is obtained, instead of doing on anonymous fd (similar to write)",
+        },
+        { .key  = {NULL} }
+
+};
diff --git a/xlators/performance/quick-read/src/Makefile.am b/xlators/performance/quick-read/src/Makefile.am
index db917f897..4906f408a 100644
--- a/xlators/performance/quick-read/src/Makefile.am
+++ b/xlators/performance/quick-read/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = quick-read.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-quick_read_la_LDFLAGS = -module -avoidversion 
+quick_read_la_LDFLAGS = -module -avoid-version 
 
 quick_read_la_SOURCES = quick-read.c
 quick_read_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = quick-read.h quick-read-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/quick-read/src/quick-read-mem-types.h b/xlators/performance/quick-read/src/quick-read-mem-types.h
index 68e893a6f..78547f641 100644
--- a/xlators/performance/quick-read/src/quick-read-mem-types.h
+++ b/xlators/performance/quick-read/src/quick-read-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __QR_MEM_TYPES_H__
@@ -24,11 +15,13 @@
 
 enum gf_qr_mem_types_ {
         gf_qr_mt_qr_inode_t   = gf_common_mt_end + 1,
+	gf_qr_mt_content_t,
         gf_qr_mt_qr_fd_ctx_t,
         gf_qr_mt_iovec,
         gf_qr_mt_qr_conf_t,
         gf_qr_mt_qr_priority_t,
         gf_qr_mt_qr_private_t,
+        gf_qr_mt_qr_unlink_ctx_t,
         gf_qr_mt_end
 };
 #endif
diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c
index a64e63de4..402da886f 100644
--- a/xlators/performance/quick-read/src/quick-read.c
+++ b/xlators/performance/quick-read/src/quick-read.c
@@ -1,177 +1,113 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "quick-read.h"
 #include "statedump.h"
 
-#define QR_DEFAULT_CACHE_SIZE 134217728
+qr_inode_t *qr_inode_ctx_get (xlator_t *this, inode_t *inode);
+void __qr_inode_prune (qr_inode_table_t *table, qr_inode_t *qr_inode);
 
-struct volume_options options[];
 
-void
-qr_local_free (qr_local_t *local)
+int
+__qr_inode_ctx_set (xlator_t *this, inode_t *inode, qr_inode_t *qr_inode)
 {
-        if (local == NULL) {
-                goto out;
-        }
-
-        if (local->stub != NULL) {
-                call_stub_destroy (local->stub);
-        }
+	uint64_t    value = 0;
+	int         ret = -1;
 
-        if (local->path != NULL) {
-                GF_FREE (local->path);
-        }
+	value = (long) qr_inode;
 
-        mem_put (local);
+	ret = __inode_ctx_set (inode, this, &value);
 
-out:
-        return;
+	return ret;
 }
 
 
-int32_t
-qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags);
-
-
-static void
-qr_loc_wipe (loc_t *loc)
+qr_inode_t *
+__qr_inode_ctx_get (xlator_t *this, inode_t *inode)
 {
-        if (loc == NULL) {
-                goto out;
-        }
-
-        if (loc->path) {
-                GF_FREE ((char *)loc->path);
-                loc->path = NULL;
-        }
+	qr_inode_t *qr_inode = NULL;
+	uint64_t    value = 0;
+	int         ret = -1;
 
-        if (loc->inode) {
-                inode_unref (loc->inode);
-                loc->inode = NULL;
-        }
+	ret = __inode_ctx_get (inode, this, &value);
+	if (ret)
+		return NULL;
 
-        if (loc->parent) {
-                inode_unref (loc->parent);
-                loc->parent = NULL;
-        }
+	qr_inode = (void *) ((long) value);
 
-out:
-        return;
+	return qr_inode;
 }
 
 
-static int32_t
-qr_loc_fill (loc_t *loc, inode_t *inode, char *path)
+qr_inode_t *
+qr_inode_ctx_get (xlator_t *this, inode_t *inode)
 {
-        int32_t  ret    = -1;
+	qr_inode_t *qr_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quick-read", loc, out, errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quick-read", inode, out, errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quick-read", path, out, errno, EINVAL);
+	LOCK (&inode->lock);
+	{
+		qr_inode = __qr_inode_ctx_get (this, inode);
+	}
+	UNLOCK (&inode->lock);
 
-        loc->inode = inode_ref (inode);
-        uuid_copy (loc->gfid, inode->gfid);
-
-        loc->path = gf_strdup (path);
-        if (!loc->path)
-                goto out;
-
-        ret = 0;
-out:
-        if (ret == -1) {
-                qr_loc_wipe (loc);
-        }
-
-        return ret;
+	return qr_inode;
 }
 
 
-void
-qr_resume_pending_ops (qr_fd_ctx_t *qr_fd_ctx, int32_t op_ret, int32_t op_errno)
+qr_inode_t *
+qr_inode_new (xlator_t *this, inode_t *inode)
 {
-        call_stub_t      *stub        = NULL, *tmp = NULL;
-        struct list_head  waiting_ops = {0, };
-
-        GF_VALIDATE_OR_GOTO ("quick-read", qr_fd_ctx, out);
-
-        INIT_LIST_HEAD (&waiting_ops);
-
-        LOCK (&qr_fd_ctx->lock);
-        {
-                qr_fd_ctx->open_in_transit = 0;
-                list_splice_init (&qr_fd_ctx->waiting_ops,
-                                  &waiting_ops);
-        }
-        UNLOCK (&qr_fd_ctx->lock);
-
-        if (!list_empty (&waiting_ops)) {
-                list_for_each_entry_safe (stub, tmp, &waiting_ops, list) {
-                        list_del_init (&stub->list);
-                        if (op_ret < 0) {
-                                qr_local_t *local = NULL;
-
-                                local = stub->frame->local;
-                                local->op_ret = op_ret;
-                                local->op_errno = op_errno;
-                        }
-
-                        call_resume (stub);
-                }
-        }
-
-out:
-        return;
-}
-
+        qr_inode_t    *qr_inode = NULL;
 
-static void
-qr_fd_ctx_free (qr_fd_ctx_t *qr_fd_ctx)
-{
-        GF_VALIDATE_OR_GOTO ("quick-read", qr_fd_ctx, out);
+        qr_inode = GF_CALLOC (1, sizeof (*qr_inode), gf_qr_mt_qr_inode_t);
+        if (!qr_inode)
+                return NULL;
 
-        GF_ASSERT (list_empty (&qr_fd_ctx->waiting_ops));
+        INIT_LIST_HEAD (&qr_inode->lru);
 
-        GF_FREE (qr_fd_ctx->path);
-        GF_FREE (qr_fd_ctx);
+        qr_inode->priority = 0; /* initial priority */
 
-out:
-        return;
+        return qr_inode;
 }
 
 
-static inline uint32_t
-is_match (const char *path, const char *pattern)
-{
-        int32_t  ret = 0;
-        uint32_t match = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", path, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", pattern, out);
-
-        ret = fnmatch (pattern, path, FNM_NOESCAPE);
-        match = (ret == 0);
+qr_inode_t *
+qr_inode_ctx_get_or_new (xlator_t *this, inode_t *inode)
+{
+	qr_inode_t   *qr_inode = NULL;
+	int           ret = -1;
+	qr_private_t *priv = NULL;
+
+	priv = this->private;
+
+	LOCK (&inode->lock);
+	{
+		qr_inode = __qr_inode_ctx_get (this, inode);
+		if (qr_inode)
+			goto unlock;
+
+		qr_inode = qr_inode_new (this, inode);
+		if (!qr_inode)
+			goto unlock;
+
+		ret = __qr_inode_ctx_set (this, inode, qr_inode);
+		if (ret) {
+			__qr_inode_prune (&priv->table, qr_inode);
+			GF_FREE (qr_inode);
+                        qr_inode = NULL;
+		}
+	}
+unlock:
+	UNLOCK (&inode->lock);
 
-out:
-        return match;
+	return qr_inode;
 }
 
 
@@ -181,2961 +117,578 @@ qr_get_priority (qr_conf_t *conf, const char *path)
         uint32_t            priority = 0;
         struct qr_priority *curr     = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", conf, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", path, out);
-
         list_for_each_entry (curr, &conf->priority_list, list) {
-                if (is_match (path, curr->pattern))
+                if (fnmatch (curr->pattern, path, FNM_NOESCAPE) == 0)
                         priority = curr->priority;
         }
 
-out:
         return priority;
 }
 
 
-/* To be called with this-priv->table.lock held */
-qr_inode_t *
-__qr_inode_alloc (xlator_t *this, char *path, inode_t *inode)
+void
+__qr_inode_register (qr_inode_table_t *table, qr_inode_t *qr_inode)
 {
-        qr_inode_t    *qr_inode = NULL;
-        qr_private_t  *priv     = NULL;
-        int            priority = 0;
+	if (!qr_inode->data)
+		return;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, path, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+	if (list_empty (&qr_inode->lru))
+		/* first time addition of this qr_inode into table */
+		table->cache_used += qr_inode->size;
+	else
+		list_del_init (&qr_inode->lru);
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        qr_inode = GF_CALLOC (1, sizeof (*qr_inode), gf_qr_mt_qr_inode_t);
-        if (qr_inode == NULL) {
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&qr_inode->lru);
-
-        priority = qr_get_priority (&priv->conf, path);
-
-        list_add_tail (&qr_inode->lru, &priv->table.lru[priority]);
-
-        qr_inode->inode = inode;
-        qr_inode->priority = priority;
-out:
-        return qr_inode;
+	list_add_tail (&qr_inode->lru, &table->lru[qr_inode->priority]);
 }
 
 
-/* To be called with qr_inode->table->lock held */
 void
-__qr_inode_free (qr_inode_t *qr_inode)
+qr_inode_set_priority (xlator_t *this, inode_t *inode, const char *path)
 {
-        GF_VALIDATE_OR_GOTO ("quick-read", qr_inode, out);
+	uint32_t          priority = 0;
+	qr_inode_table_t *table = NULL;
+	qr_inode_t       *qr_inode = NULL;
+        qr_private_t     *priv = NULL;
+	qr_conf_t        *conf = NULL;
 
-        if (qr_inode->xattr) {
-                dict_unref (qr_inode->xattr);
-        }
+	qr_inode = qr_inode_ctx_get (this, inode);
+	if (!qr_inode)
+		return;
 
-        list_del (&qr_inode->lru);
+	priv = this->private;
+	table = &priv->table;
+	conf = &priv->conf;
 
-        GF_FREE (qr_inode);
-out:
-        return;
-}
-
-/* To be called with priv->table.lock held */
-void
-__qr_cache_prune (xlator_t *this)
-{
-        qr_private_t     *priv          = NULL;
-        qr_conf_t        *conf          = NULL;
-        qr_inode_table_t *table         = NULL;
-        qr_inode_t        *curr         = NULL, *next = NULL;
-        int32_t           index         = 0;
-        uint64_t          size_to_prune = 0;
-        uint64_t          size_pruned   = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+	if (path)
+		priority = qr_get_priority (conf, path);
+	else
+		/* retain existing priority, just bump LRU */
+		priority = qr_inode->priority;
 
-        table = &priv->table;
-        conf = &priv->conf;
+	LOCK (&table->lock);
+	{
+		qr_inode->priority = priority;
 
-        size_to_prune = table->cache_used - conf->cache_size;
-
-        for (index=0; index < conf->max_pri; index++) {
-                list_for_each_entry_safe (curr, next, &table->lru[index], lru) {
-                        size_pruned += curr->stbuf.ia_size;
-                        inode_ctx_del (curr->inode, this, NULL);
-                        __qr_inode_free (curr);
-                        if (size_pruned >= size_to_prune)
-                                goto done;
-                }
-        }
-
-done:
-        table->cache_used -= size_pruned;
-
-out:
-        return;
-}
-
-/* To be called with table->lock held */
-inline char
-__qr_need_cache_prune (qr_conf_t *conf, qr_inode_table_t *table)
-{
-        char need_prune = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", conf, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", table, out);
-
-        need_prune = (table->cache_used > conf->cache_size);
-
-out:
-        return need_prune;
+		__qr_inode_register (table, qr_inode);
+	}
+	UNLOCK (&table->lock);
 }
 
 
-int32_t
-qr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *dict, struct iatt *postparent)
+/* To be called with priv->table.lock held */
+void
+__qr_inode_prune (qr_inode_table_t *table, qr_inode_t *qr_inode)
 {
-        data_t           *content  = NULL;
-        qr_inode_t       *qr_inode = NULL;
-        uint64_t          value    = 0;
-        int               ret      = -1;
-        qr_conf_t        *conf     = NULL;
-        qr_inode_table_t *table    = NULL;
-        qr_private_t     *priv     = NULL;
-        qr_local_t       *local    = NULL;
-
-        GF_ASSERT (frame);
-
-        if ((op_ret == -1) || (dict == NULL)) {
-                goto out;
-        }
-
-        if ((this == NULL) || (this->private == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "quick-read configuration is not found");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        priv = this->private;
-        conf = &priv->conf;
-        table = &priv->table;
+	GF_FREE (qr_inode->data);
+	qr_inode->data = NULL;
 
-        local = frame->local;
+	if (!list_empty (&qr_inode->lru)) {
+		table->cache_used -= qr_inode->size;
+		qr_inode->size = 0;
 
-        if (buf->ia_size > conf->max_file_size) {
-                goto out;
-        }
+		list_del_init (&qr_inode->lru);
+	}
 
-        if (IA_ISDIR (buf->ia_type)) {
-                goto out;
-        }
-
-        if (inode == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "lookup returned a NULL inode");
-                goto out;
-        }
-
-        content = dict_get (dict, GF_CONTENT_KEY);
-        if (content == NULL) {
-                goto out;
-        }
-
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (inode, this, &value);
-                if (ret == -1) {
-                        qr_inode = __qr_inode_alloc (this, local->path, inode);
-                        if (qr_inode == NULL) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unlock;
-                        }
-
-                        ret = inode_ctx_put (inode, this,
-                                             (uint64_t)(long)qr_inode);
-                        if (ret == -1) {
-                                __qr_inode_free (qr_inode);
-                                qr_inode = NULL;
-                                op_ret = -1;
-                                op_errno = EINVAL;
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot set quick-read context in "
-                                        "inode (gfid:%s)",
-                                        uuid_utoa (inode->gfid));
-                                goto unlock;
-                        }
-                } else {
-                        qr_inode = (qr_inode_t *)(long)value;
-                        if (qr_inode == NULL) {
-                                op_ret = -1;
-                                op_errno = EINVAL;
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot find quick-read context in "
-                                        "inode (gfid:%s)",
-                                        uuid_utoa (inode->gfid));
-                                goto unlock;
-                        }
-                }
-
-                if (qr_inode->xattr) {
-                        dict_unref (qr_inode->xattr);
-                        qr_inode->xattr = NULL;
-                        table->cache_used -= qr_inode->stbuf.ia_size;
-                }
-
-                qr_inode->xattr = dict_ref (dict);
-                qr_inode->stbuf = *buf;
-                table->cache_used += buf->ia_size;
-
-                gettimeofday (&qr_inode->tv, NULL);
-                if (__qr_need_cache_prune (conf, table)) {
-                        __qr_cache_prune (this);
-                }
-        }
-unlock:
-        UNLOCK (&table->lock);
-
-out:
-        /*
-         * FIXME: content size in dict can be greater than the size application
-         * requested for. Applications need to be careful till this is fixed.
-         */
-        QR_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf, dict,
-                         postparent);
-
-        return 0;
+	memset (&qr_inode->buf, 0, sizeof (qr_inode->buf));
 }
 
 
-int32_t
-qr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+void
+qr_inode_prune (xlator_t *this, inode_t *inode)
 {
-        qr_conf_t        *conf           = NULL;
-        dict_t           *new_req_dict   = NULL;
-        int32_t           op_ret         = -1, op_errno = EINVAL;
-        data_t           *content        = NULL;
-        uint64_t          requested_size = 0, size = 0, value = 0;
-        char              cached         = 0;
-        qr_inode_t       *qr_inode       = NULL;
-        qr_private_t     *priv           = NULL;
-        qr_inode_table_t *table          = NULL;
-        qr_local_t       *local          = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (frame->this->name, priv, unwind);
-
-        conf = &priv->conf;
-        if (conf == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        table = &priv->table;
-        local = mem_get0 (this->local_pool);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, unwind, op_errno,
-                                        ENOMEM);
-
-        frame->local = local;
-
-        local->path = gf_strdup (loc->path);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, unwind, op_errno,
-                                        ENOMEM);
-        LOCK (&table->lock);
-        {
-                op_ret = inode_ctx_get (loc->inode, this, &value);
-                if (op_ret == 0) {
-                        qr_inode = (qr_inode_t *)(long)value;
-                        if (qr_inode != NULL) {
-                                if (qr_inode->xattr) {
-                                        cached = 1;
-                                }
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        if ((xattr_req == NULL) && (conf->max_file_size > 0)) {
-                new_req_dict = xattr_req = dict_new ();
-                if (xattr_req == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-        }
-
-        if (!cached) {
-                if (xattr_req) {
-                        content = dict_get (xattr_req, GF_CONTENT_KEY);
-                        if (content) {
-                                requested_size = data_to_uint64 (content);
-                        }
-                }
-
-                if ((conf->max_file_size > 0)
-                    && (conf->max_file_size != requested_size)) {
-                        size = (conf->max_file_size > requested_size) ?
-                                conf->max_file_size : requested_size;
-
-                        op_ret = dict_set (xattr_req, GF_CONTENT_KEY,
-                                           data_from_uint64 (size));
-                        if (op_ret < 0) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot set key in request dict to "
-                                        "request file "
-                                        "content during lookup cbk");
-                                goto unwind;
-                        }
-                }
-        }
-
-        STACK_WIND (frame, qr_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-
-        if (new_req_dict) {
-                dict_unref (new_req_dict);
-        }
-
-        return 0;
+        qr_private_t     *priv          = NULL;
+        qr_inode_table_t *table         = NULL;
+	qr_inode_t       *qr_inode      = NULL;
 
-unwind:
-        QR_STACK_UNWIND (lookup, frame, op_ret, op_errno, NULL, NULL, NULL,
-                         NULL);
+	qr_inode = qr_inode_ctx_get (this, inode);
+	if (!qr_inode)
+		return;
 
-        if (new_req_dict) {
-                dict_unref (new_req_dict);
-        }
+	priv = this->private;
+	table = &priv->table;
 
-        return 0;
+	LOCK (&table->lock);
+	{
+		__qr_inode_prune (table, qr_inode);
+	}
+	UNLOCK (&table->lock);
 }
 
 
-int32_t
-qr_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd)
+/* To be called with priv->table.lock held */
+void
+__qr_cache_prune (qr_inode_table_t *table, qr_conf_t *conf)
 {
-        uint64_t          value     = 0;
-        int32_t           ret       = -1;
-        qr_local_t       *local     = NULL;
-        qr_inode_t       *qr_inode  = NULL;
-        qr_fd_ctx_t      *qr_fd_ctx = NULL;
-        call_stub_t      *stub      = NULL, *tmp = NULL;
-        char              is_open   = 0;
-        qr_private_t     *priv      = NULL;
-        qr_inode_table_t *table     = NULL;
-        struct list_head  waiting_ops;
-
-        GF_ASSERT (frame);
-
-        priv = this->private;
-        table = &priv->table;
-
-        local = frame->local;
-        if (local != NULL) {
-                is_open = local->is_open;
-        }
-
-        INIT_LIST_HEAD (&waiting_ops);
-
-        ret = fd_ctx_get (fd, this, &value);
-        if ((ret == -1) && (op_ret != -1)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot find quick-read context in fd (%p) opened on "
-                        "inode (gfid: %s)", fd, uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        if (value) {
-                qr_fd_ctx = (qr_fd_ctx_t *) (long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        qr_fd_ctx->open_in_transit = 0;
+        qr_inode_t        *curr = NULL;
+	qr_inode_t        *next = NULL;
+        int                index = 0;
+	size_t             size_pruned = 0;
 
-                        if (op_ret == 0) {
-                                qr_fd_ctx->opened = 1;
-                        }
-                        list_splice_init (&qr_fd_ctx->waiting_ops,
-                                          &waiting_ops);
-                }
-                UNLOCK (&qr_fd_ctx->lock);
-
-                if (local && local->is_open
-                    && ((local->open_flags & O_TRUNC) == O_TRUNC)) {
-                        LOCK (&table->lock);
-                        {
-                                ret = inode_ctx_del (fd->inode, this, &value);
-                                if (ret == 0) {
-                                        qr_inode = (qr_inode_t *)(long) value;
-
-                                        if (qr_inode != NULL) {
-                                                __qr_inode_free (qr_inode);
-                                        }
-                                }
-                        }
-                        UNLOCK (&table->lock);
-                }
+        for (index = 0; index < conf->max_pri; index++) {
+                list_for_each_entry_safe (curr, next, &table->lru[index], lru) {
 
-                if (!list_empty (&waiting_ops)) {
-                        list_for_each_entry_safe (stub, tmp, &waiting_ops,
-                                                  list) {
-                                list_del_init (&stub->list);
-                                if (op_ret < 0) {
-                                        qr_local_t *local = NULL;
+                        size_pruned += curr->size;
 
-                                        local = stub->frame->local;
-                                        local->op_ret = op_ret;
-                                        local->op_errno = op_errno;
-                                }
+                        __qr_inode_prune (table, curr);
 
-                                call_resume (stub);
-                        }
+                        if (table->cache_used < conf->cache_size)
+				return;
                 }
         }
-out:
-        if (is_open) {
-                QR_STACK_UNWIND (open, frame, op_ret, op_errno, fd);
-        } else {
-                STACK_DESTROY (frame->root);
-        }
 
-        return 0;
+        return;
 }
 
 
-int32_t
-qr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+void
+qr_cache_prune (xlator_t *this)
 {
-        qr_inode_t       *qr_inode       = NULL;
-        int32_t           ret            = -1;
-        uint64_t          filep          = 0;
-        char              content_cached = 0;
-        qr_fd_ctx_t      *qr_fd_ctx      = NULL, *tmp_fd_ctx = NULL;
-        int32_t           op_ret         = -1, op_errno = EINVAL;
-        qr_local_t       *local          = NULL;
-        qr_private_t     *priv           = NULL;
-        qr_inode_table_t *table          = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this->private, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+        qr_private_t      *priv = NULL;
+        qr_conf_t         *conf = NULL;
+        qr_inode_table_t  *table = NULL;
 
         priv = this->private;
         table = &priv->table;
+        conf = &priv->conf;
 
-        tmp_fd_ctx = qr_fd_ctx = GF_CALLOC (1, sizeof (*qr_fd_ctx),
-                                            gf_qr_mt_qr_fd_ctx_t);
-        if (qr_fd_ctx == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        LOCK_INIT (&qr_fd_ctx->lock);
-        INIT_LIST_HEAD (&qr_fd_ctx->waiting_ops);
-
-        qr_fd_ctx->path = gf_strdup (loc->path);
-        if (qr_fd_ctx->path == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        qr_fd_ctx->flags = flags;
-        qr_fd_ctx->wbflags = wbflags;
-
-        ret = fd_ctx_set (fd, this, (uint64_t)(long)qr_fd_ctx);
-        if (ret == -1) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot set quick-read context in "
-                        "fd (%p) opened on inode (gfid:%s)", fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        tmp_fd_ctx = NULL;
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        local->is_open = 1;
-        local->open_flags = flags;
-        frame->local = local;
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (fd->inode, this, &filep);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long) filep;
-                        if (qr_inode) {
-                                if (qr_inode->xattr) {
-                                        content_cached = 1;
-                                }
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        if (content_cached && (flags & O_DIRECTORY)) {
-                op_ret = -1;
-                op_errno = ENOTDIR;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open with O_DIRECTORY flag received on non-directory");
-                goto unwind;
-        }
-
-        if (!content_cached || ((flags & O_ACCMODE) == O_WRONLY)
-            || ((flags & O_TRUNC) == O_TRUNC)
-            || ((flags & O_DIRECT) == O_DIRECT)
-            || ((wbflags & GF_OPEN_NOWB) != 0)) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        /*
-                         * we really need not set this flag, since open is
-                         * not yet unwound.
-                         */
-
-                        qr_fd_ctx->open_in_transit = 1;
-                        if (((flags & O_DIRECT) == O_DIRECT)
-                            || ((wbflags & GF_OPEN_NOWB)) != 0) {
-                                qr_fd_ctx->disabled = 1;
-                        }
-                }
-                UNLOCK (&qr_fd_ctx->lock);
-                goto wind;
-        } else {
-                op_ret = 0;
-                op_errno = 0;
-                goto unwind;
-        }
-
-unwind:
-        if (tmp_fd_ctx != NULL) {
-                qr_fd_ctx_free (tmp_fd_ctx);
-        }
-
-        QR_STACK_UNWIND (open, frame, op_ret, op_errno, fd);
-        return 0;
-
-wind:
-        STACK_WIND (frame, qr_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
-        return 0;
-}
-
-
-static inline time_t
-qr_time_elapsed (struct timeval *now, struct timeval *then)
-{
-        time_t time_elapsed = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", now, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", then, out);
-
-        time_elapsed = now->tv_sec - then->tv_sec;
-
-out:
-        return time_elapsed;
+	LOCK (&table->lock);
+	{
+		if (table->cache_used > conf->cache_size)
+			__qr_cache_prune (table, conf);
+	}
+	UNLOCK (&table->lock);
 }
 
 
-static inline char
-qr_need_validation (qr_conf_t *conf, qr_inode_t *qr_inode)
+void *
+qr_content_extract (dict_t *xdata)
 {
-        struct timeval now             = {0, };
-        char           need_validation = 0;
+	data_t  *data = NULL;
+	void    *content = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", conf, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", qr_inode, out);
+	data = dict_get (xdata, GF_CONTENT_KEY);
+	if (!data)
+		return NULL;
 
-        gettimeofday (&now, NULL);
+	content = GF_CALLOC (1, data->len, gf_qr_mt_content_t);
+	if (!content)
+		return NULL;
 
-        if (qr_time_elapsed (&now, &qr_inode->tv) >= conf->cache_timeout)
-                need_validation = 1;
+	memcpy (content, data->data, data->len);
 
-out:
-        return need_validation;
+	return content;
 }
 
 
-static int32_t
-qr_validate_cache_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *buf)
+void
+qr_content_update (xlator_t *this, qr_inode_t *qr_inode, void *data,
+		   struct iatt *buf)
 {
-        qr_inode_t       *qr_inode  = NULL;
-        qr_local_t       *local     = NULL;
-        uint64_t          value     = 0;
-        int32_t           ret       = 0;
-        qr_private_t     *priv      = NULL;
-        qr_inode_table_t *table     = NULL;
-        call_stub_t      *stub      = NULL;
-
-        GF_ASSERT (frame);
-        if (this == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "xlator object (this) is NULL");
-                goto unwind;
-        }
-
-        local = frame->local;
-        if ((local == NULL) || ((local->fd) == NULL)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (local == NULL) ? "local is NULL"
-                        : "fd is not stored in local");
-                goto unwind;
-        }
-
-        local->just_validated = 1;
-
-        if (op_ret == -1) {
-                goto unwind;
-        }
+        qr_private_t      *priv = NULL;
+        qr_inode_table_t  *table = NULL;
 
         priv = this->private;
         table = &priv->table;
 
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (local->fd->inode, this, &value);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long) value;
-                }
-
-                if (qr_inode != NULL) {
-                        gettimeofday (&qr_inode->tv, NULL);
-
-                        if ((qr_inode->stbuf.ia_mtime != buf->ia_mtime)
-                            || (qr_inode->stbuf.ia_mtime_nsec
-                                != buf->ia_mtime_nsec)) {
-                                inode_ctx_del (local->fd->inode, this, NULL);
-                                __qr_inode_free (qr_inode);
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        stub = local->stub;
-        local->stub = NULL;
-
-        call_resume (stub);
-
-        return 0;
-
-unwind:
-        /* this is actually unwind of readv */
-        QR_STACK_UNWIND (readv, frame, op_ret, op_errno, NULL, -1, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-qr_validate_cache_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        qr_local_t *local  = NULL;
-        int32_t     op_ret = -1, op_errno = -1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, out);
-
-        local = frame->local;
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        } else {
-                op_ret = local->op_ret;
-                op_errno = local->op_errno;
-        }
-
-out:
-        if (op_ret == -1) {
-                qr_validate_cache_cbk (frame, NULL, this, op_ret, op_errno,
-                                       NULL);
-        } else {
-                STACK_WIND (frame, qr_validate_cache_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fstat, fd);
-        }
-
-        return 0;
-}
-
-
-int
-qr_validate_cache (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   call_stub_t *stub)
-{
-        int           ret           = -1;
-        int           flags         = 0;
-        uint64_t      value         = 0;
-        loc_t         loc           = {0, };
-        char         *path          = NULL;
-        qr_local_t   *local         = NULL;
-        qr_fd_ctx_t  *qr_fd_ctx     = NULL;
-        call_stub_t  *validate_stub = NULL;
-        char          need_open     = 0, can_wind = 0, validate_cbk_called = 0;
-        call_frame_t *open_frame    = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, stub, out);
-
-        if (frame->local == NULL) {
-                local = mem_get0 (this->local_pool);
-                if (local == NULL) {
-                        goto out;
-                }
-        } else {
-                local = frame->local;
-        }
-
-        local->fd = fd;
-        local->stub = stub;
-        frame->local = local;
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                validate_stub = fop_fstat_stub (frame,
-                                                                qr_validate_cache_helper,
-                                                                fd);
-                                if (validate_stub == NULL) {
-                                        ret = -1;
-                                        if (need_open) {
-                                                qr_fd_ctx->open_in_transit = 0;
-                                        }
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&validate_stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-
-                if (ret == -1) {
-                        goto out;
-                }
-        } else {
-                can_wind = 1;
-        }
+	LOCK (&table->lock);
+	{
+		__qr_inode_prune (table, qr_inode);
 
-        if (need_open) {
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        validate_cbk_called = 1;
-                        goto out;
-                }
+		qr_inode->data = data;
+		qr_inode->size = buf->ia_size;
 
-                ret = qr_loc_fill (&loc, fd->inode, path);
-                if (ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        validate_cbk_called = 1;
-                        STACK_DESTROY (open_frame->root);
-                        goto out;
-                }
+		qr_inode->ia_mtime = buf->ia_mtime;
+		qr_inode->ia_mtime_nsec = buf->ia_mtime_nsec;
 
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open,
-                            &loc, flags, fd, qr_fd_ctx->wbflags);
+		qr_inode->buf = *buf;
 
-                qr_loc_wipe (&loc);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_validate_cache_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fstat, fd);
-        }
+		gettimeofday (&qr_inode->last_refresh, NULL);
 
-        ret = 0;
-out:
-        if ((ret < 0) && !validate_cbk_called) {
-                if (frame->local == NULL) {
-                        call_stub_destroy (stub);
-                }
+		__qr_inode_register (table, qr_inode);
+	}
+	UNLOCK (&table->lock);
 
-                qr_validate_cache_cbk (frame, NULL, this, -1, errno, NULL);
-        }
-        return ret;
+	qr_cache_prune (this);
 }
 
 
-int32_t
-qr_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iovec *vector, int32_t count,
-              struct iatt *stbuf, struct iobref *iobref)
+gf_boolean_t
+qr_size_fits (qr_conf_t *conf, struct iatt *buf)
 {
-        GF_ASSERT (frame);
-
-        QR_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                         stbuf, iobref);
-        return 0;
+	return (buf->ia_size <= conf->max_file_size);
 }
 
 
-int32_t
-qr_readv_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset, uint32_t flags)
+gf_boolean_t
+qr_mtime_equal (qr_inode_t *qr_inode, struct iatt *buf)
 {
-        qr_local_t  *local    = NULL;
-        int32_t      op_errno = EINVAL, ret = 0;
-        uint64_t     value    = 0;
-        qr_fd_ctx_t *fdctx   = NULL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding read call",
-                        fdctx ? fdctx->path : NULL, strerror (errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_readv_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readv, fd, size, offset, flags);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
-        return 0;
+	return (qr_inode->ia_mtime == buf->ia_mtime &&
+		qr_inode->ia_mtime_nsec == buf->ia_mtime_nsec);
 }
 
 
-int32_t
-qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t read_flags)
+void
+__qr_content_refresh (xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf)
 {
-        qr_inode_t        *qr_inode       = NULL;
-        int32_t            ret            = -1, op_ret = -1, op_errno = -1;
-        uint64_t           value          = 0;
-        int                count          = -1, flags = 0, i = 0;
-        char               content_cached = 0, need_validation = 0;
-        char               need_open      = 0, can_wind = 0, need_unwind = 0;
-        struct iobuf      *iobuf          = NULL;
-        struct iobref     *iobref         = NULL;
-        struct iatt        stbuf          = {0, };
-        data_t            *content        = NULL;
-        qr_fd_ctx_t       *qr_fd_ctx      = NULL;
-        call_stub_t       *stub           = NULL;
-        loc_t              loc            = {0, };
-        qr_conf_t         *conf           = NULL;
-        struct iovec      *vector         = NULL;
-        char              *path           = NULL;
-        off_t              start          = 0, end = 0;
-        size_t             len            = 0;
-        struct iobuf_pool *iobuf_pool     = NULL;
-        qr_local_t        *local          = NULL;
-        char               just_validated = 0;
-        qr_private_t      *priv           = NULL;
-        qr_inode_table_t  *table          = NULL;
-        call_frame_t      *open_frame     = NULL;
-
-        op_ret = 0;
+        qr_private_t      *priv = NULL;
+        qr_inode_table_t  *table = NULL;
+	qr_conf_t         *conf = NULL;
 
         priv = this->private;
-        conf = &priv->conf;
         table = &priv->table;
+	conf = &priv->conf;
 
-        local = frame->local;
-
-        if (local != NULL) {
-                just_validated = local->just_validated;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-                if (qr_fd_ctx != NULL) {
-                        if (qr_fd_ctx->disabled) {
-                                goto out;
-                        }
-                }
-        }
-
-        iobuf_pool = this->ctx->iobuf_pool;
-
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (fd->inode, this, &value);
-                if (ret)
-                        goto unlock;
-
-                qr_inode = (qr_inode_t *)(long)value;
-                if (!qr_inode || !qr_inode->xattr)
-                        goto unlock;
-
-                if (!just_validated
-                    && qr_need_validation (conf, qr_inode)) {
-                        need_validation = 1;
-                        goto unlock;
-                }
-
-                content = dict_get (qr_inode->xattr, GF_CONTENT_KEY);
-
-                stbuf = qr_inode->stbuf;
-                content_cached = 1;
-                list_move_tail (&qr_inode->lru,
-                                &table->lru[qr_inode->priority]);
+	if (qr_size_fits (conf, buf) && qr_mtime_equal (qr_inode, buf)) {
+		qr_inode->buf = *buf;
 
-                if (offset > content->len) {
-                        op_ret = 0;
-                        end = content->len;
-                } else {
-                        if ((offset + size) > content->len) {
-                                op_ret = content->len - offset;
-                                end = content->len;
-                        } else {
-                                op_ret = size;
-                                end =  offset + size;
-                        }
-                }
-
-                count = (op_ret  / iobuf_pool->default_page_size);
-                if ((op_ret % iobuf_pool->default_page_size) != 0) {
-                        count++;
-                }
-
-                if (count == 0) {
-                        op_ret = 0;
-                        goto unlock;
-                }
-
-                vector = GF_CALLOC (count, sizeof (*vector), gf_qr_mt_iovec);
-                if (vector == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        need_unwind = 1;
-                        goto unlock;
-                }
-
-                iobref = iobref_new ();
-                if (iobref == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        need_unwind = 1;
-                        goto unlock;
-                }
-
-                for (i = 0; i < count; i++) {
-                        /* TODO: Now that we have support for variable
-                           io-buf-sizes, i guess we need to get rid of
-                           default size here */
-                        iobuf = iobuf_get (iobuf_pool);
-                        if (iobuf == NULL) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                need_unwind = 1;
-                                goto unlock;
-                        }
-
-                        start = offset + (iobuf_pool->default_page_size * i);
+		gettimeofday (&qr_inode->last_refresh, NULL);
 
-                        if (start > end) {
-                                len = 0;
-                        } else {
-                                len = (iobuf_pool->default_page_size >
-                                       ((end - start)) ? (end - start) :
-                                       iobuf_pool->default_page_size);
-
-                                memcpy (iobuf->ptr, content->data + start, len);
-                        }
+		__qr_inode_register (table, qr_inode);
+	} else {
+		__qr_inode_prune (table, qr_inode);
+	}
 
-                        iobref_add (iobref, iobuf);
-                        iobuf_unref (iobuf);
-
-                        vector[i].iov_base = iobuf->ptr;
-                        vector[i].iov_len = len;
-                }
-        }
-unlock:
-        UNLOCK (&table->lock);
-
-out:
-        if (content_cached || need_unwind) {
-                QR_STACK_UNWIND (readv, frame, op_ret, op_errno, vector,
-                                 count, &stbuf, iobref);
-
-        } else if (need_validation) {
-                stub = fop_readv_stub (frame, qr_readv, fd, size, offset,
-                                       read_flags);
-                if (stub == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                qr_validate_cache (frame, this, fd, stub);
-        } else {
-                if (qr_fd_ctx) {
-                        LOCK (&qr_fd_ctx->lock);
-                        {
-                                path = qr_fd_ctx->path;
-                                flags = qr_fd_ctx->flags;
-
-                                if (!(qr_fd_ctx->opened
-                                      || qr_fd_ctx->open_in_transit)) {
-                                        need_open = 1;
-                                        qr_fd_ctx->open_in_transit = 1;
-                                }
-
-                                if (qr_fd_ctx->opened) {
-                                        can_wind = 1;
-                                } else {
-                                        if (frame->local == NULL) {
-                                                frame->local = mem_get0 (this->local_pool);
-                                                if (frame->local == NULL) {
-                                                        op_ret = -1;
-                                                        op_errno = ENOMEM;
-                                                        need_unwind = 1;
-                                                        qr_fd_ctx->open_in_transit = 0;
-                                                        goto fdctx_unlock;
-                                                }
-                                        }
-
-                                        stub = fop_readv_stub (frame,
-                                                               qr_readv_helper,
-                                                               fd, size,
-                                                               offset,
-                                                               read_flags);
-                                        if (stub == NULL) {
-                                                op_ret = -1;
-                                                op_errno = ENOMEM;
-                                                need_unwind = 1;
-                                                qr_fd_ctx->open_in_transit = 0;
-                                                goto fdctx_unlock;
-                                        }
-
-                                        list_add_tail (&stub->list,
-                                                       &qr_fd_ctx->waiting_ops);
-                                }
-                        }
-                fdctx_unlock:
-                        UNLOCK (&qr_fd_ctx->lock);
-
-                        if (op_ret == -1) {
-                                need_unwind = 1;
-                                goto out;
-                        }
-                } else {
-                        can_wind = 1;
-                }
-
-                if (need_open) {
-                        op_ret = qr_loc_fill (&loc, fd->inode, path);
-                        if (op_ret == -1) {
-                                qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                                goto ret;
-                        }
-
-                        open_frame = create_frame (this, this->ctx->pool);
-                        if (open_frame == NULL) {
-                                qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                                qr_loc_wipe (&loc);
-                                goto ret;
-                        }
-
-                        STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->open,
-                                    &loc, flags, fd, qr_fd_ctx->wbflags);
-
-                        qr_loc_wipe (&loc);
-                } else if (can_wind) {
-                        STACK_WIND (frame, qr_readv_cbk, FIRST_CHILD (this),
-                                    FIRST_CHILD (this)->fops->readv, fd, size,
-                                    offset, read_flags);
-                }
-        }
-
-ret:
-        if (vector) {
-                GF_FREE (vector);
-        }
-
-        if (iobref) {
-                iobref_unref (iobref);
-        }
-
-        return 0;
+	return;
 }
 
 
-int32_t
-qr_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
-        return 0;
-}
-
-
-int32_t
-qr_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  struct iovec *vector, int32_t count, off_t off,
-                  uint32_t flags, struct iobref *iobref)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding write call",
-                        fdctx ? fdctx->path : NULL, strerror (errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_writev_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev, fd, vector, count, off,
-                    flags, iobref);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-qr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t wr_flags, struct iobref *iobref)
+void
+qr_content_refresh (xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf)
 {
-        uint64_t          value      = 0;
-        int               flags      = 0;
-        call_stub_t      *stub       = NULL;
-        char             *path       = NULL;
-        loc_t             loc        = {0, };
-        qr_inode_t       *qr_inode   = NULL;
-        qr_fd_ctx_t      *qr_fd_ctx  = NULL;
-        int32_t           op_ret     = -1, op_errno = -1, ret = -1;
-        char              can_wind   = 0, need_unwind = 0, need_open = 0;
-        qr_private_t     *priv       = NULL;
-        qr_inode_table_t *table      = NULL;
-        call_frame_t     *open_frame = NULL;
+        qr_private_t      *priv = NULL;
+        qr_inode_table_t  *table = NULL;
 
         priv = this->private;
         table = &priv->table;
 
-        ret = fd_ctx_get (fd, this, &value);
-
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (fd->inode, this, &value);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long)value;
-                        if (qr_inode != NULL) {
-                                inode_ctx_del (fd->inode, this, NULL);
-                                __qr_inode_free (qr_inode);
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_writev_stub (frame, qr_writev_helper,
-                                                        fd, vector, count, off,
-                                                        wr_flags, iobref);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-        if (need_unwind) {
-                QR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_writev_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->writev, fd, vector, count,
-                            off, wr_flags, iobref);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *buf)
-{
-        QR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
-        return 0;
-}
-
-
-int32_t
-qr_fstat_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fstat call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_fstat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fstat, fd);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-qr_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        uint64_t      value      = 0;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        call_stub_t  *stub       = NULL;
-        loc_t         loc        = {0, };
-        char         *path       = NULL;
-        int           flags      = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto unwind;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fstat_stub (frame, qr_fstat_helper,
-                                                       fd);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-unwind:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fstat, frame, op_ret, op_errno, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fstat_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fstat, fd);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
+	LOCK (&table->lock);
+	{
+		__qr_content_refresh (this, qr_inode, buf);
+	}
+	UNLOCK (&table->lock);
 }
 
 
-int32_t
-qr_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preop, struct iatt *postop)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, preop, postop);
-        return 0;
-}
-
-
-int32_t
-qr_fsetattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fsetattr "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_fsetattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf,
-                    valid);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-qr_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid)
-{
-        uint64_t      value      = 0;
-        int           flags      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL" :
-                        "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fsetattr_stub (frame,
-                                                          qr_fsetattr_helper,
-                                                          fd, stbuf, valid);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, NULL, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fsetattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetattr, fd, stbuf,
-                            valid);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int32_t
-qr_fsetxattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     dict_t *dict, int32_t flags)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fsetxattr "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_fsetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fsetxattr, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-qr_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags)
-{
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        int           open_flags = 0;
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) "
-                        "is NULL" : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        open_flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fsetxattr_stub (frame,
-                                                           qr_fsetxattr_helper,
-                                                           fd, dict, flags);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fsetxattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetxattr, fd, dict,
-                            flags);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, open_flags,
-                            fd, qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict);
-        return 0;
-}
-
-
-int32_t
-qr_fgetxattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name)
+gf_boolean_t
+__qr_cache_is_fresh (xlator_t *this, qr_inode_t *qr_inode)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+	qr_conf_t        *conf = NULL;
+	qr_private_t     *priv = NULL;
+	struct timeval    now;
+	struct timeval    diff;
 
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
+	priv = this->private;
+	conf = &priv->conf;
 
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
+	gettimeofday (&now, NULL);
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fgetxattr "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
+	timersub (&now, &qr_inode->last_refresh, &diff);
 
-        STACK_WIND (frame, qr_fgetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fgetxattr, fd, name);
-        return 0;
+	if (diff.tv_sec >= conf->cache_timeout)
+		return _gf_false;
 
-unwind:
-        QR_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL);
-        return 0;
+	return _gf_true;
 }
 
 
-int32_t
-qr_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name)
-{
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        /*
-         * FIXME: Can quick-read use the extended attributes stored in the
-         * cache? this needs to be discussed.
-         */
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL" :
-                        "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fgetxattr_stub (frame,
-                                                           qr_fgetxattr_helper,
-                                                           fd, name);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (open, frame, op_ret, op_errno, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fgetxattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fgetxattr, fd, name);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (flush, frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int32_t
-qr_flush_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
+int
+qr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode_ret,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding flush call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_flush_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->flush, fd);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (flush, frame, -1, op_errno);
-        return 0;
-}
+        void             *content  = NULL;
+        qr_inode_t       *qr_inode = NULL;
+	inode_t          *inode    = NULL;
 
+	inode = frame->local;
+	frame->local = NULL;
 
-int32_t
-qr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        uint64_t     value     = 0;
-        call_stub_t *stub      = NULL;
-        qr_fd_ctx_t *qr_fd_ctx = NULL;
-        int32_t      ret       = -1, op_ret = -1, op_errno = EINVAL;
-        char         can_wind  = 0, need_unwind = 0;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
+        if (op_ret == -1) {
+		qr_inode_prune (this, inode);
                 goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else if (qr_fd_ctx->open_in_transit) {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_flush_stub (frame, qr_flush_helper,
-                                                       fd);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        } else {
-                                op_ret = 0;
-                                need_unwind = 1;
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
+	}
+
+	if (dict_get (xdata, "sh-failed")) {
+		qr_inode_prune (this, inode);
+		goto out;
+	}
+
+	content = qr_content_extract (xdata);
+
+	if (content) {
+		/* new content came along, always replace old content */
+		qr_inode = qr_inode_ctx_get_or_new (this, inode);
+		if (!qr_inode) {
+			/* no harm done */
+			GF_FREE (content);
+			goto out;
+		}
+		qr_content_update (this, qr_inode, content, buf);
+	} else {
+		/* purge old content if necessary */
+		qr_inode = qr_inode_ctx_get (this, inode);
+		if (!qr_inode)
+			/* usual path for large files */
+			goto out;
+
+		qr_content_refresh (this, qr_inode, buf);
+	}
 out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (flush, frame, op_ret, op_errno);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_flush_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->flush, fd);
-        }
-
-        return 0;
-}
+	if (inode)
+		inode_unref (inode);
 
-
-int32_t
-qr_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode_ret,
+			     buf, xdata, postparent);
         return 0;
 }
 
 
-int32_t
-qr_fentrylk_helper (call_frame_t *frame, xlator_t *this, const char *volume,
-                    fd_t *fd, const char *basename, entrylk_cmd cmd,
-                    entrylk_type type)
-{
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fentrylk "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND(frame, qr_fentrylk_cbk, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename,
-                   cmd, type);
-        return 0;
-
-unwind:
-        QR_STACK_UNWIND (fentrylk, frame, -1, op_errno);
-        return 0;
-}
-
-
-int32_t
-qr_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-             const char *basename, entrylk_cmd cmd, entrylk_type type)
+int
+qr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fentrylk_stub (frame,
-                                                          qr_fentrylk_helper,
-                                                          volume, fd, basename,
-                                                          cmd, type);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fentrylk_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fentrylk, volume, fd,
-                            basename, cmd, type);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
+        qr_private_t     *priv           = NULL;
+        qr_conf_t        *conf           = NULL;
+        qr_inode_t       *qr_inode       = NULL;
+	int               ret            = -1;
+	dict_t           *new_xdata      = NULL;
 
-                qr_loc_wipe (&loc);
-        }
+        priv = this->private;
+        conf = &priv->conf;
 
-ret:
-        return 0;
-}
+	qr_inode = qr_inode_ctx_get (this, loc->inode);
+	if (qr_inode && qr_inode->data)
+		/* cached. only validate in qr_lookup_cbk */
+		goto wind;
+
+	if (!xdata)
+		xdata = new_xdata = dict_new ();
+
+	if (!xdata)
+		goto wind;
+
+	ret = 0;
+	if (conf->max_file_size)
+		ret = dict_set (xdata, GF_CONTENT_KEY,
+				data_from_uint64 (conf->max_file_size));
+	if (ret)
+		gf_log (this->name, GF_LOG_WARNING,
+			"cannot set key in request dict (%s)",
+			loc->path);
+wind:
+	frame->local = inode_ref (loc->inode);
 
+        STACK_WIND (frame, qr_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-int32_t
-qr_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
+	if (new_xdata)
+		dict_unref (new_xdata);
 
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
         return 0;
 }
 
 
-int32_t
-qr_finodelk_helper (call_frame_t *frame, xlator_t *this, const char *volume,
-                    fd_t *fd, int32_t cmd, struct gf_flock *lock)
+int
+qr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
+        gf_dirent_t *entry      = NULL;
+	qr_inode_t  *qr_inode   = NULL;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+	if (op_ret <= 0)
+		goto unwind;
 
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
+        list_for_each_entry (entry, &entries->list, list) {
+                if (!entry->inode)
+			continue;
 
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
+		qr_inode = qr_inode_ctx_get (this, entry->inode);
+		if (!qr_inode)
+			/* no harm */
+			continue;
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding finodelk "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
+		qr_content_refresh (this, qr_inode, &entry->d_stat);
         }
 
-        STACK_WIND (frame, qr_finodelk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, lock);
-        return 0;
-
 unwind:
-        QR_STACK_UNWIND (finodelk, frame, -1, op_errno);
-        return 0;
+	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-             int32_t cmd, struct gf_flock *lock)
-{
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_finodelk_stub (frame,
-                                                          qr_finodelk_helper,
-                                                          volume, fd, cmd,
-                                                          lock);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_finodelk_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->finodelk, volume, fd,
-                            cmd, lock);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf)
+int
+qr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     size_t size, off_t offset, dict_t *xdata)
 {
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf);
-        return 0;
+	STACK_WIND (frame, qr_readdirp_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readdirp,
+		    fd, size, offset, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_fsync_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+int
+qr_readv_cached (call_frame_t *frame, qr_inode_t *qr_inode, size_t size,
+		 off_t offset, uint32_t flags, dict_t *xdata)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
+	xlator_t         *this = NULL;
+	qr_private_t     *priv = NULL;
+	qr_inode_table_t *table = NULL;
+	int               op_ret = -1;
+	struct iobuf     *iobuf = NULL;
+	struct iobref    *iobref = NULL;
+	struct iovec      iov = {0, };
+	struct iatt       buf = {0, };
 
-        GF_ASSERT (frame);
+	this = frame->this;
+	priv = this->private;
+	table = &priv->table;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+	LOCK (&table->lock);
+	{
+		op_ret = -1;
 
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
+		if (!qr_inode->data)
+			goto unlock;
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding fsync call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
+		if (offset >= qr_inode->size)
+			goto unlock;
 
-        STACK_WIND (frame, qr_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags);
-        return 0;
+		if (!__qr_cache_is_fresh (this, qr_inode))
+			goto unlock;
 
-unwind:
-        QR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
+		op_ret = min (size, (qr_inode->size - offset));
 
+		iobuf = iobuf_get2 (this->ctx->iobuf_pool, op_ret);
+		if (!iobuf) {
+			op_ret = -1;
+			goto unlock;
+		}
 
-int32_t
-qr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        int           open_flags = 0;
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
+		iobref = iobref_new ();
+		if (!iobref) {
+			op_ret = -1;
+			goto unlock;
+		}
 
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
+		iobref_add (iobref, iobuf);
 
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        open_flags = qr_fd_ctx->flags;
+		memcpy (iobuf->ptr, qr_inode->data + offset, op_ret);
 
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
+		buf = qr_inode->buf;
 
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_fsync_stub (frame, qr_fsync_helper,
-                                                       fd, flags);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
+		/* bump LRU */
+		__qr_inode_register (table, qr_inode);
+	}
+unlock:
+	UNLOCK (&table->lock);
 
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (fsync, frame, op_ret, op_errno, NULL, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_fsync_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsync, fd, flags);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
+	if (op_ret > 0) {
+		iov.iov_base = iobuf->ptr;
+		iov.iov_len = op_ret;
 
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
+		STACK_UNWIND_STRICT (readv, frame, op_ret, 0, &iov, 1,
+				     &buf, iobref, xdata);
+	}
 
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, open_flags,
-                            fd, qr_fd_ctx->wbflags);
+	if (iobuf)
+		iobuf_unref (iobuf);
 
-                qr_loc_wipe (&loc);
-        }
+	if (iobref)
+		iobref_unref (iobref);
 
-ret:
-        return 0;
+	return op_ret;
 }
 
 
-int32_t
-qr_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-        int32_t           ret      = 0;
-        uint64_t          value    = 0;
-        qr_inode_t       *qr_inode = NULL;
-        qr_local_t       *local    = NULL;
-        qr_private_t     *priv     = NULL;
-        qr_inode_table_t *table    = NULL;
-
-        GF_ASSERT (frame);
-
-        if (op_ret == -1) {
-                goto out;
-        }
-
-        local = frame->local;
-        if ((local == NULL) || (local->fd == NULL)
-            || (local->fd->inode == NULL)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (frame->this->name, GF_LOG_WARNING, "cannot get inode");
-                goto out;
-        }
-
-        if ((this == NULL) || (this->private == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "cannot get quick read configuration from xlator "
-                        "object");
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        priv = this->private;
-        table = &priv->table;
-
-        LOCK (&table->lock);
-        {
-                ret = inode_ctx_get (local->fd->inode, this, &value);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long) value;
-
-                        if (qr_inode) {
-                                if (qr_inode->stbuf.ia_size != postbuf->ia_size)
-                                {
-                                        inode_ctx_del (local->fd->inode, this,
-                                                       NULL);
-                                        __qr_inode_free (qr_inode);
-                                }
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-out:
-        QR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf,
-                         postbuf);
-        return 0;
-}
-
-
-int32_t
-qr_ftruncate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     off_t offset)
+int
+qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
+	qr_inode_t *qr_inode = NULL;
 
-        GF_ASSERT (frame);
+	qr_inode = qr_inode_ctx_get (this, fd->inode);
+	if (!qr_inode)
+		goto wind;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding ftruncate "
-                        "call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        return 0;
+	if (qr_readv_cached (frame, qr_inode, size, offset, flags, xdata) <= 0)
+		goto wind;
 
-unwind:
-        QR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
-        return 0;
+	return 0;
+wind:
+	STACK_WIND (frame, default_readv_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
+		    fd, size, offset, flags, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+int
+qr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
+	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
+	   dict_t *xdata)
 {
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_local_t   *local      = NULL;
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                need_unwind = 1;
-                goto out;
-        }
-
-        local->fd = fd;
-        frame->local = local;
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
+	qr_inode_prune (this, fd->inode);
 
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                stub = fop_ftruncate_stub (frame,
-                                                           qr_ftruncate_helper,
-                                                           fd, offset);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
-
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL,
-                                 NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_ftruncate_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
-}
-
-
-int32_t
-qr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-           int32_t op_errno, struct gf_flock *lock)
-{
-        GF_ASSERT (frame);
-        QR_STACK_UNWIND (lk, frame, op_ret, op_errno, lock);
-        return 0;
+	STACK_WIND (frame, default_writev_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
+		    fd, iov, count, offset, flags, iobref, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_lk_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-              struct gf_flock *lock)
+int
+qr_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+	     dict_t *xdata)
 {
-        qr_local_t  *local    = NULL;
-        qr_fd_ctx_t *fdctx    = NULL;
-        uint64_t     value    = 0;
-        int32_t      ret      = 0;
-        int32_t      op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-
-                ret = fd_ctx_get (fd, this, &value);
-                if (ret == 0) {
-                        fdctx = (qr_fd_ctx_t *)(long) value;
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "open failed on path (%s) (%s), unwinding lk call",
-                        fdctx ? fdctx->path : NULL, strerror (op_errno));
-                goto unwind;
-        }
-
-        STACK_WIND (frame, qr_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk, fd, cmd, lock);
-
-        return 0;
+	qr_inode_prune (this, loc->inode);
 
-unwind:
-        QR_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
-        return 0;
+	STACK_WIND (frame, default_truncate_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
+		    loc, offset, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-       struct gf_flock *lock)
+int
+qr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	      dict_t *xdata)
 {
-        int           flags      = 0;
-        uint64_t      value      = 0;
-        call_stub_t  *stub       = NULL;
-        char         *path       = NULL;
-        loc_t         loc        = {0, };
-        qr_fd_ctx_t  *qr_fd_ctx  = NULL;
-        int32_t       ret        = -1, op_ret = -1, op_errno = EINVAL;
-        char          need_open  = 0, can_wind = 0, need_unwind = 0;
-        call_frame_t *open_frame = NULL;
-
-        GF_ASSERT (frame);
-        if ((this == NULL) || (fd == NULL)) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        (this == NULL) ? "xlator object (this) is NULL"
-                        : "fd is NULL");
-                need_unwind = 1;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long)value;
-        }
-
-        if (qr_fd_ctx) {
-                LOCK (&qr_fd_ctx->lock);
-                {
-                        path = qr_fd_ctx->path;
-                        flags = qr_fd_ctx->flags;
-
-                        if (!(qr_fd_ctx->opened
-                              || qr_fd_ctx->open_in_transit)) {
-                                need_open = 1;
-                                qr_fd_ctx->open_in_transit = 1;
-                        }
-
-                        if (qr_fd_ctx->opened) {
-                                can_wind = 1;
-                        } else {
-                                frame->local = mem_get0 (this->local_pool);
-                                if (frame->local == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                stub = fop_lk_stub (frame, qr_lk_helper, fd,
-                                                    cmd, lock);
-                                if (stub == NULL) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        need_unwind = 1;
-                                        qr_fd_ctx->open_in_transit = 0;
-                                        goto unlock;
-                                }
-
-                                list_add_tail (&stub->list,
-                                               &qr_fd_ctx->waiting_ops);
-                        }
-                }
-        unlock:
-                UNLOCK (&qr_fd_ctx->lock);
-        } else {
-                can_wind = 1;
-        }
+	qr_inode_prune (this, fd->inode);
 
-out:
-        if (need_unwind) {
-                QR_STACK_UNWIND (lk, frame, op_ret, op_errno, NULL);
-        } else if (can_wind) {
-                STACK_WIND (frame, qr_lk_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lk, fd, cmd, lock);
-        } else if (need_open) {
-                op_ret = qr_loc_fill (&loc, fd->inode, path);
-                if (op_ret == -1) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, errno);
-                        goto ret;
-                }
-
-                open_frame = create_frame (this, this->ctx->pool);
-                if (open_frame == NULL) {
-                        qr_resume_pending_ops (qr_fd_ctx, -1, ENOMEM);
-                        qr_loc_wipe (&loc);
-                        goto ret;
-                }
-
-                STACK_WIND (open_frame, qr_open_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open, &loc, flags, fd,
-                            qr_fd_ctx->wbflags);
-
-                qr_loc_wipe (&loc);
-        }
-
-ret:
-        return 0;
+	STACK_WIND (frame, default_ftruncate_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
+		    fd, offset, xdata);
+	return 0;
 }
 
 
-int32_t
-qr_release (xlator_t *this, fd_t *fd)
+int
+qr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	 fd_t *fd, dict_t *xdata)
 {
-        qr_fd_ctx_t *qr_fd_ctx = NULL;
-        int32_t      ret       = 0;
-        uint64_t     value     = 0;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        ret = fd_ctx_del (fd, this, &value);
-        if (ret == 0) {
-                qr_fd_ctx = (qr_fd_ctx_t *)(long) value;
-                if (qr_fd_ctx) {
-                        qr_fd_ctx_free (qr_fd_ctx);
-                }
-        }
+	qr_inode_set_priority (this, fd->inode, loc->path);
 
-out:
-        return 0;
+	STACK_WIND (frame, default_open_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->open,
+		    loc, flags, fd, xdata);
+	return 0;
 }
 
-
-int32_t
+int
 qr_forget (xlator_t *this, inode_t *inode)
 {
         qr_inode_t   *qr_inode = NULL;
-        uint64_t      value     = 0;
-        int32_t       ret       = -1;
-        qr_private_t *priv      = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+	qr_inode = qr_inode_ctx_get (this, inode);
 
-        priv = this->private;
+	if (!qr_inode)
+		return 0;
 
-        LOCK (&priv->table.lock);
-        {
-                ret = inode_ctx_del (inode, this, &value);
-                if (ret == 0) {
-                        qr_inode = (qr_inode_t *)(long) value;
-                        __qr_inode_free (qr_inode);
-                }
-        }
-        UNLOCK (&priv->table.lock);
+	qr_inode_prune (this, inode);
 
-out:
-        return 0;
+	GF_FREE (qr_inode);
+
+	return 0;
 }
 
 
@@ -3143,101 +696,35 @@ int32_t
 qr_inodectx_dump (xlator_t *this, inode_t *inode)
 {
         qr_inode_t *qr_inode = NULL;
-        uint64_t    value    = 0;
         int32_t     ret      = -1;
         char        key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
         char        buf[256]                        = {0, };
-        struct tm  *tm                              = NULL;
-        ret = inode_ctx_get (inode, this, &value);
-        if (ret != 0) {
-                goto out;
-        }
 
-        qr_inode = (qr_inode_t *)(long)value;
-        if (qr_inode == NULL) {
+        qr_inode = qr_inode_ctx_get (this, inode);
+        if (!qr_inode)
                 goto out;
-        }
 
         gf_proc_dump_build_key (key_prefix, "xlator.performance.quick-read",
                                 "inodectx");
         gf_proc_dump_add_section (key_prefix);
 
-        gf_proc_dump_write ("entire-file-cached", "%s", qr_inode->xattr ? "yes" : "no");
-
-        tm = localtime (&qr_inode->tv.tv_sec);
-        strftime (buf, 256, "%Y-%m-%d %H:%M:%S", tm);
-        snprintf (buf + strlen (buf), 256 - strlen (buf),
-                  ".%"GF_PRI_SUSECONDS, qr_inode->tv.tv_usec);
-
-        gf_proc_dump_write ("last-cache-validation-time", "%s", buf);
-
-        ret = 0;
-out:
-        return ret;
-}
+        gf_proc_dump_write ("entire-file-cached", "%s", qr_inode->data ? "yes" : "no");
 
-int32_t
-qr_fdctx_dump (xlator_t *this, fd_t *fd)
-{
-        qr_fd_ctx_t *fdctx = NULL;
-        uint64_t     value = 0;
-        int32_t      ret   = 0, i = 0;
-        char         key[GF_DUMP_MAX_BUF_LEN]        = {0, };
-        char         key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        call_stub_t *stub                            = NULL;
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret != 0) {
-                goto out;
-        }
+        if (qr_inode->last_refresh.tv_sec) {
+                gf_time_fmt (buf, sizeof buf, qr_inode->last_refresh.tv_sec,
+                             gf_timefmt_FT);
+                snprintf (buf + strlen (buf), sizeof buf - strlen (buf),
+                          ".%"GF_PRI_SUSECONDS, qr_inode->last_refresh.tv_usec);
 
-        fdctx = (qr_fd_ctx_t *)(long)value;
-        if (fdctx == NULL) {
-                goto out;
+                gf_proc_dump_write ("last-cache-validation-time", "%s", buf);
         }
 
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.quick-read",
-                                "fdctx");
-        gf_proc_dump_add_section (key_prefix);
-
-        gf_proc_dump_write ("fd", "%p", fd);
-
-        gf_proc_dump_write ("path", "%s", fdctx->path);
-
-        LOCK (&fdctx->lock);
-        {
-                gf_proc_dump_write ("opened", "%s", fdctx->opened ? "yes" : "no");
-
-                gf_proc_dump_write ("open-in-progress", "%s", fdctx->open_in_transit ?
-                                    "yes" : "no");
-
-                gf_proc_dump_write ("caching disabled (for this fd)", "%s",
-                                    fdctx->disabled ? "yes" : "no");
-
-                gf_proc_dump_write ("flags", "%d", fdctx->flags);
-
-                gf_proc_dump_write ("wbflags", "%d", fdctx->wbflags);
-
-                list_for_each_entry (stub, &fdctx->waiting_ops, list) {
-                        gf_proc_dump_build_key (key, "",
-                                                "waiting-ops[%d].frame", i);
-                        gf_proc_dump_write (key, "%"PRId64,
-                                            stub->frame->root->unique);
-
-                        gf_proc_dump_build_key (key, "",
-                                                "waiting-ops[%d].fop", i);
-                        gf_proc_dump_write (key, "%s", gf_fop_list[stub->fop]);
-
-                        i++;
-                }
-        }
-        UNLOCK (&fdctx->lock);
-
         ret = 0;
 out:
         return ret;
 }
 
+
 int
 qr_priv_dump (xlator_t *this)
 {
@@ -3257,14 +744,11 @@ qr_priv_dump (xlator_t *this)
         priv = this->private;
         conf = &priv->conf;
 
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING, "conf null in xlator");
+        if (!conf)
                 return -1;
-        }
 
         table = &priv->table;
 
-
         gf_proc_dump_build_key (key_prefix, "xlator.performance.quick-read",
                                 "priv");
 
@@ -3274,13 +758,12 @@ qr_priv_dump (xlator_t *this)
         gf_proc_dump_write ("cache_timeout", "%d", conf->cache_timeout);
 
         if (!table) {
-                gf_log (this->name, GF_LOG_WARNING, "table is NULL");
                 goto out;
         } else {
                 for (i = 0; i < conf->max_pri; i++) {
                         list_for_each_entry (curr, &table->lru[i], lru) {
                                 file_count++;
-                                total_size += curr->stbuf.ia_size;
+                                total_size += curr->size;
                         }
                 }
         }
@@ -3312,7 +795,8 @@ mem_acct_init (xlator_t *this)
         return ret;
 }
 
-gf_boolean_t
+
+static gf_boolean_t
 check_cache_size_ok (xlator_t *this, int64_t cache_size)
 {
         int                     ret = _gf_true;
@@ -3335,7 +819,7 @@ check_cache_size_ok (xlator_t *this, int64_t cache_size)
         else
                 max_cache_size = total_mem;
 
-        gf_log (this->name, GF_LOG_INFO, "Max cache size is %"PRIu64,
+        gf_log (this->name, GF_LOG_DEBUG, "Max cache size is %"PRIu64,
                 max_cache_size);
         if (cache_size > max_cache_size) {
                 ret = _gf_false;
@@ -3355,6 +839,7 @@ reconfigure (xlator_t *this, dict_t *options)
         qr_private_t *priv           = NULL;
         qr_conf_t    *conf           = NULL;
         uint64_t       cache_size_new = 0;
+
         GF_VALIDATE_OR_GOTO ("quick-read", this, out);
         GF_VALIDATE_OR_GOTO (this->name, this->private, out);
         GF_VALIDATE_OR_GOTO (this->name, options, out);
@@ -3466,13 +951,9 @@ qr_get_priority_list (const char *opt_str, struct list_head *first)
                 priority_str = strtok_r (NULL, ",", &tmp_str);
         }
 out:
-        if (string != NULL) {
-                GF_FREE (string);
-        }
+        GF_FREE (string);
 
-        if (dup_str != NULL) {
-                GF_FREE (dup_str);
-        }
+        GF_FREE (dup_str);
 
         if (max_pri == -1) {
                 list_for_each_entry_safe (curr, tmp, first, list) {
@@ -3552,14 +1033,6 @@ init (xlator_t *this)
                 INIT_LIST_HEAD (&priv->table.lru[i]);
         }
 
-        this->local_pool = mem_pool_new (qr_local_t, 64);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
         ret = 0;
 
         this->private = priv;
@@ -3573,37 +1046,79 @@ out:
 
 
 void
+qr_inode_table_destroy (qr_private_t *priv)
+{
+        int        i    = 0;
+        qr_conf_t *conf = NULL;
+
+        conf = &priv->conf;
+
+        for (i = 0; i < conf->max_pri; i++) {
+                GF_ASSERT (list_empty (&priv->table.lru[i]));
+        }
+
+        LOCK_DESTROY (&priv->table.lock);
+
+        return;
+}
+
+
+void
+qr_conf_destroy (qr_conf_t *conf)
+{
+        struct qr_priority *curr = NULL, *tmp = NULL;
+
+        list_for_each_entry_safe (curr, tmp, &conf->priority_list, list) {
+                list_del (&curr->list);
+                GF_FREE (curr->pattern);
+                GF_FREE (curr);
+        }
+
+        return;
+}
+
+
+void
 fini (xlator_t *this)
 {
+        qr_private_t *priv = NULL;
+
+        if (this == NULL) {
+                goto out;
+        }
+
+        priv = this->private;
+        if (priv == NULL) {
+                goto out;
+        }
+
+        qr_inode_table_destroy (priv);
+        qr_conf_destroy (&priv->conf);
+
+        this->private = NULL;
+
+        GF_FREE (priv);
+out:
         return;
 }
 
 struct xlator_fops fops = {
         .lookup      = qr_lookup,
+	.readdirp    = qr_readdirp,
         .open        = qr_open,
         .readv       = qr_readv,
-        .writev      = qr_writev,
-        .fstat       = qr_fstat,
-        .fsetxattr   = qr_fsetxattr,
-        .fgetxattr   = qr_fgetxattr,
-        .flush       = qr_flush,
-        .fentrylk    = qr_fentrylk,
-        .finodelk    = qr_finodelk,
-        .fsync       = qr_fsync,
-        .ftruncate   = qr_ftruncate,
-        .lk          = qr_lk,
-        .fsetattr    = qr_fsetattr,
+	.writev      = qr_writev,
+	.truncate    = qr_truncate,
+	.ftruncate   = qr_ftruncate
 };
 
 struct xlator_cbks cbks = {
         .forget  = qr_forget,
-        .release = qr_release,
 };
 
 struct xlator_dumpops dumpops = {
         .priv      =  qr_priv_dump,
         .inodectx  =  qr_inodectx_dump,
-        .fdctx     =  qr_fdctx_dump
 };
 
 struct volume_options options[] = {
@@ -3629,4 +1144,5 @@ struct volume_options options[] = {
           .max  = 1 * GF_UNIT_KB * 1000,
           .default_value = "64KB",
         },
+        { .key  = {NULL} }
 };
diff --git a/xlators/performance/quick-read/src/quick-read.h b/xlators/performance/quick-read/src/quick-read.h
index 064151b63..6f0a05417 100644
--- a/xlators/performance/quick-read/src/quick-read.h
+++ b/xlators/performance/quick-read/src/quick-read.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __QUICK_READ_H
@@ -43,40 +34,20 @@
 #include <fnmatch.h>
 #include "quick-read-mem-types.h"
 
-struct qr_fd_ctx {
-        char              opened;
-        char              disabled;
-        char              open_in_transit;
-        char             *path;
-        int               flags;
-        int               wbflags;
-        struct list_head  waiting_ops;
-        gf_lock_t         lock;
-};
-typedef struct qr_fd_ctx qr_fd_ctx_t;
-
-struct qr_local {
-        char         is_open;
-        char        *path;
-        char         just_validated;
-        fd_t        *fd;
-        int          open_flags;
-        int32_t      op_ret;
-        int32_t      op_errno;
-        call_stub_t *stub;
-};
-typedef struct qr_local qr_local_t;
 
 struct qr_inode {
-        dict_t           *xattr;
-        inode_t          *inode;
+	void             *data;
+	size_t            size;
         int               priority;
-        struct iatt       stbuf;
-        struct timeval    tv;
+	uint32_t          ia_mtime;
+	uint32_t          ia_mtime_nsec;
+	struct iatt       buf;
+        struct timeval    last_refresh;
         struct list_head  lru;
 };
 typedef struct qr_inode qr_inode_t;
 
+
 struct qr_priority {
         char             *pattern;
         int32_t           priority;
@@ -106,13 +77,5 @@ struct qr_private {
 };
 typedef struct qr_private qr_private_t;
 
-void qr_local_free (qr_local_t *local);
-
-#define QR_STACK_UNWIND(op, frame, params ...) do {             \
-                qr_local_t *__local = frame->local;             \
-                frame->local = NULL;                            \
-                STACK_UNWIND_STRICT (op, frame, params);        \
-                qr_local_free (__local);                        \
-        } while (0)
 
 #endif /* #ifndef __QUICK_READ_H */
diff --git a/xlators/performance/read-ahead/src/Makefile.am b/xlators/performance/read-ahead/src/Makefile.am
index b46020aac..be80ae7ac 100644
--- a/xlators/performance/read-ahead/src/Makefile.am
+++ b/xlators/performance/read-ahead/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = read-ahead.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-read_ahead_la_LDFLAGS = -module -avoidversion
+read_ahead_la_LDFLAGS = -module -avoid-version
 
 read_ahead_la_SOURCES = read-ahead.c page.c
 read_ahead_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = read-ahead.h read-ahead-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/read-ahead/src/page.c b/xlators/performance/read-ahead/src/page.c
index 1e6a92087..6e5b52c5e 100644
--- a/xlators/performance/read-ahead/src/page.c
+++ b/xlators/performance/read-ahead/src/page.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -142,7 +133,8 @@ ra_waitq_return (ra_waitq_t *waitq)
 int
 ra_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int32_t op_ret, int32_t op_errno, struct iovec *vector,
-              int32_t count, struct iatt *stbuf, struct iobref *iobref)
+              int32_t count, struct iatt *stbuf, struct iobref *iobref,
+              dict_t *xdata)
 {
         ra_local_t   *local          = NULL;
         off_t         pending_offset = 0;
@@ -278,7 +270,7 @@ ra_page_fault (ra_file_t *file, call_frame_t *frame, off_t offset)
         STACK_WIND (fault_frame, ra_fault_cbk,
                     FIRST_CHILD (fault_frame->this),
                     FIRST_CHILD (fault_frame->this)->fops->readv,
-                    file->fd, file->page_size, offset, 0);
+                    file->fd, file->page_size, offset, 0, NULL);
 
         return;
 
@@ -429,7 +421,12 @@ ra_frame_unwind (call_frame_t *frame)
                                 fill->count * sizeof (*vector));
 
                         copied += (fill->count * sizeof (*vector));
-                        iobref_merge (iobref, fill->iobref);
+                        if (iobref_merge (iobref, fill->iobref)) {
+				local->op_ret = -1;
+				local->op_errno = ENOMEM;
+				iobref_unref (iobref);
+				iobref = NULL;
+			}
                 }
 
                 fill->next->prev = fill->prev;
@@ -447,7 +444,7 @@ ra_frame_unwind (call_frame_t *frame)
         file = (ra_file_t *)(long)tmp_file;
 
         STACK_UNWIND_STRICT (readv, frame, local->op_ret, local->op_errno,
-                             vector, count, &file->stbuf, iobref);
+                             vector, count, &file->stbuf, iobref, NULL);
 
         iobref_unref (iobref);
         pthread_mutex_destroy (&local->local_lock);
@@ -508,6 +505,9 @@ ra_page_wakeup (ra_page_t *page)
                 ra_frame_fill (page, frame);
         }
 
+        if (page->stale) {
+                ra_page_purge (page);
+        }
 out:
         return waitq;
 }
diff --git a/xlators/performance/read-ahead/src/read-ahead-mem-types.h b/xlators/performance/read-ahead/src/read-ahead-mem-types.h
index d2d184f09..219e29289 100644
--- a/xlators/performance/read-ahead/src/read-ahead-mem-types.h
+++ b/xlators/performance/read-ahead/src/read-ahead-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
diff --git a/xlators/performance/read-ahead/src/read-ahead.c b/xlators/performance/read-ahead/src/read-ahead.c
index 1a4dda97a..6e2d84591 100644
--- a/xlators/performance/read-ahead/src/read-ahead.c
+++ b/xlators/performance/read-ahead/src/read-ahead.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /*
@@ -44,12 +35,11 @@ read_ahead (call_frame_t *frame, ra_file_t *file);
 
 int
 ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd)
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         ra_conf_t  *conf    = NULL;
         ra_file_t  *file    = NULL;
         int         ret     = 0;
-        long        wbflags = 0;
 
         GF_ASSERT (frame);
         GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
@@ -60,8 +50,6 @@ ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto unwind;
         }
 
-        wbflags = (long)frame->local;
-
         file = GF_CALLOC (1, sizeof (*file), gf_ra_mt_ra_file_t);
         if (!file) {
                 op_ret = -1;
@@ -74,10 +62,6 @@ ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((fd->flags & O_DIRECT) || ((fd->flags & O_ACCMODE) == O_WRONLY))
                 file->disabled = 1;
 
-        if (wbflags & GF_OPEN_NOWB) {
-                file->disabled = 1;
-        }
-
         file->offset = (unsigned long long) 0;
         file->conf = conf;
         file->pages.next = &file->pages;
@@ -116,7 +100,7 @@ ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unwind:
         frame->local = NULL;
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 
         return 0;
 }
@@ -126,7 +110,7 @@ int
 ra_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
+               struct iatt *postparent, dict_t *xdata)
 {
         ra_conf_t  *conf = NULL;
         ra_file_t  *file = NULL;
@@ -187,7 +171,7 @@ ra_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 unwind:
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         return 0;
 }
@@ -195,17 +179,15 @@ unwind:
 
 int
 ra_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+         fd_t *fd, dict_t *xdata)
 {
         GF_ASSERT (frame);
         GF_ASSERT (this);
 
-        frame->local = (void *)(long)wbflags;
-
         STACK_WIND (frame, ra_open_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
 
         return 0;
 }
@@ -213,7 +195,7 @@ ra_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
 
 int
 ra_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params)
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         GF_ASSERT (frame);
         GF_ASSERT (this);
@@ -221,7 +203,7 @@ ra_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         STACK_WIND (frame, ra_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
 
         return 0;
 }
@@ -248,8 +230,12 @@ flush_region (call_frame_t *frame, ra_file_t *file, off_t offset, off_t size,
                                 if (!trav->waitq) {
                                         ra_page_purge (trav);
                                 }
-                                else if (for_write) {
-                                        trav->poisoned = 1;
+                                else {
+                                        trav->stale = 1;
+
+                                        if (for_write) {
+                                                trav->poisoned = 1;
+                                        }
                                 }
                         }
                         trav = next;
@@ -358,7 +344,8 @@ out:
 int
 ra_need_atime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                   dict_t *xdata)
 {
         GF_ASSERT (frame);
         STACK_DESTROY (frame->root);
@@ -449,7 +436,7 @@ dispatch_requests (call_frame_t *frame, ra_file_t *file)
                 STACK_WIND (ra_frame, ra_need_atime_cbk,
                             FIRST_CHILD (frame->this),
                             FIRST_CHILD (frame->this)->fops->readv,
-                            file->fd, 1, 1, 0);
+                            file->fd, 1, 1, 0, NULL);
         }
 
 out:
@@ -460,12 +447,13 @@ out:
 int
 ra_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                       int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                       int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                       dict_t *xdata)
 {
         GF_ASSERT (frame);
 
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+                             stbuf, iobref, xdata);
 
         return 0;
 }
@@ -473,7 +461,7 @@ ra_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 ra_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags)
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
         ra_file_t   *file            = NULL;
         ra_local_t  *local           = NULL;
@@ -510,7 +498,7 @@ ra_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                         "expected offset (%"PRId64") when page_count=%d",
                         offset, file->page_count);
 
-                if (file->expected < (conf->page_size * conf->page_count)) {
+                if (file->expected < (file->page_size * conf->page_count)) {
                         file->expected += size;
                         file->page_count = min ((file->expected
                                                  / file->page_size),
@@ -553,7 +541,8 @@ ra_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                             NULL);
 
         return 0;
 
@@ -561,17 +550,17 @@ disabled:
         STACK_WIND (frame, ra_readv_disabled_cbk,
                     FIRST_CHILD (frame->this),
                     FIRST_CHILD (frame->this)->fops->readv,
-                    fd, size, offset, flags);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
 
 int
 ra_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno)
+              int32_t op_errno, dict_t *xdata)
 {
         GF_ASSERT (frame);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -579,16 +568,18 @@ ra_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
 
 int
 ra_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf)
+              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
         GF_ASSERT (frame);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
         return 0;
 }
 
 
 int
-ra_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+ra_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         ra_file_t *file     = NULL;
         uint64_t   tmp_file = 0;
@@ -606,17 +597,18 @@ ra_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
         }
 
         STACK_WIND (frame, ra_flush_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->flush, fd);
+                    FIRST_CHILD (this)->fops->flush, fd, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int
-ra_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
+ra_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
 {
         ra_file_t *file     = NULL;
         uint64_t   tmp_file = 0;
@@ -634,11 +626,11 @@ ra_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
         }
 
         STACK_WIND (frame, ra_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync, fd, datasync);
+                    FIRST_CHILD (this)->fops->fsync, fd, datasync, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -646,7 +638,7 @@ unwind:
 int
 ra_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
         ra_file_t *file     = NULL;
 
@@ -659,14 +651,16 @@ ra_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
         return 0;
 }
 
 
 int
 ra_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref)
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
         ra_file_t *file    = NULL;
         uint64_t  tmp_file = 0;
@@ -688,12 +682,12 @@ ra_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
         STACK_WIND (frame, ra_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
 
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -701,29 +695,30 @@ unwind:
 int
 ra_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
+                 struct iatt *postbuf, dict_t *xdata)
 {
         GF_ASSERT (frame);
 
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+                             postbuf, xdata);
         return 0;
 }
 
 
 int
 ra_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, struct iatt *buf)
+             int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         GF_ASSERT (frame);
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
-ra_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+ra_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
         ra_file_t *file     = NULL;
         fd_t      *iter_fd  = NULL;
@@ -762,11 +757,11 @@ ra_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         STACK_WIND (frame, ra_truncate_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->truncate,
-                    loc, offset);
+                    loc, offset, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -863,7 +858,7 @@ out:
 }
 
 int
-ra_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+ra_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         ra_file_t *file     = NULL;
         fd_t      *iter_fd  = NULL;
@@ -892,17 +887,18 @@ ra_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
         UNLOCK (&inode->lock);
 
         STACK_WIND (frame, ra_attr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fstat, fd);
+                    FIRST_CHILD (this)->fops->fstat, fd, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
+        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int
-ra_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+ra_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
         ra_file_t *file    = NULL;
         fd_t      *iter_fd = NULL;
@@ -938,14 +934,114 @@ ra_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         UNLOCK (&inode->lock);
 
         STACK_WIND (frame, ra_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate, fd, offset);
+                    FIRST_CHILD (this)->fops->ftruncate, fd, offset, xdata);
+        return 0;
+
+unwind:
+        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+ra_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        GF_ASSERT (frame);
+
+        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+        return 0;
+}
+
+static int
+ra_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	     size_t len, dict_t *xdata)
+{
+        ra_file_t *file    = NULL;
+        fd_t      *iter_fd = NULL;
+        inode_t   *inode   = NULL;
+        uint64_t  tmp_file = 0;
+        int32_t   op_errno = EINVAL;
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
+        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+
+        inode = fd->inode;
+
+        LOCK (&inode->lock);
+        {
+                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                        fd_ctx_get (iter_fd, this, &tmp_file);
+                        file = (ra_file_t *)(long)tmp_file;
+                        if (!file)
+                                continue;
+
+                        flush_region(frame, file, offset, len, 1);
+                }
+        }
+        UNLOCK (&inode->lock);
+
+        STACK_WIND (frame, ra_discard_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->discard, fd, offset, len, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+        STACK_UNWIND_STRICT (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+ra_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        GF_ASSERT (frame);
+
+        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
         return 0;
 }
 
+static int
+ra_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata)
+{
+        ra_file_t *file    = NULL;
+        fd_t      *iter_fd = NULL;
+        inode_t   *inode   = NULL;
+        uint64_t  tmp_file = 0;
+        int32_t   op_errno = EINVAL;
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
+        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+
+        inode = fd->inode;
+
+        LOCK (&inode->lock);
+        {
+                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                        fd_ctx_get (iter_fd, this, &tmp_file);
+                        file = (ra_file_t *)(long)tmp_file;
+                        if (!file)
+                                continue;
+
+                        flush_region(frame, file, offset, len, 1);
+                }
+        }
+        UNLOCK (&inode->lock);
+
+        STACK_WIND (frame, ra_zerofill_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->zerofill, fd,
+                    offset, len, xdata);
+        return 0;
+
+unwind:
+        STACK_UNWIND_STRICT (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
 int
 ra_priv_dump (xlator_t *this)
@@ -953,6 +1049,7 @@ ra_priv_dump (xlator_t *this)
         ra_conf_t       *conf                           = NULL;
         int             ret                             = -1;
         char            key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+        gf_boolean_t    add_section                     = _gf_false;
 
         if (!this) {
                 goto out;
@@ -964,26 +1061,32 @@ ra_priv_dump (xlator_t *this)
                 goto out;
         }
 
-        ret = pthread_mutex_trylock (&conf->conf_lock);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Unable to lock client %s "
-                        "(%s)", this->name, strerror (ret));
-                ret = -1;
-                goto out;
-        }
-
         gf_proc_dump_build_key (key_prefix, "xlator.performance.read-ahead",
                                 "priv");
 
         gf_proc_dump_add_section (key_prefix);
-        gf_proc_dump_write ("page_size", "%d", conf->page_size);
-        gf_proc_dump_write ("page_count", "%d", conf->page_count);
-        gf_proc_dump_write ("force_atime_update", "%d", conf->force_atime_update);
+        add_section = _gf_true;
 
+        ret = pthread_mutex_trylock (&conf->conf_lock);
+        if (ret)
+                goto out;
+        {
+                gf_proc_dump_write ("page_size", "%d", conf->page_size);
+                gf_proc_dump_write ("page_count", "%d", conf->page_count);
+                gf_proc_dump_write ("force_atime_update", "%d",
+                                    conf->force_atime_update);
+        }
         pthread_mutex_unlock (&conf->conf_lock);
 
         ret = 0;
 out:
+        if (ret && conf) {
+                if (add_section == _gf_false)
+                        gf_proc_dump_add_section (key_prefix);
+
+                gf_proc_dump_write ("Unable to dump priv",
+                                    "(Lock acquisition failed) %s", this->name);
+        }
         return ret;
 }
 
@@ -1021,6 +1124,8 @@ reconfigure (xlator_t *this, dict_t *options)
 
         GF_OPTION_RECONF ("page-count", conf->page_count, options, uint32, out);
 
+	GF_OPTION_RECONF ("page-size", conf->page_size, options, size, out);
+
         ret = 0;
  out:
         return ret;
@@ -1053,6 +1158,8 @@ init (xlator_t *this)
 
         conf->page_size = this->ctx->page_size;
 
+	GF_OPTION_INIT ("page-size", conf->page_size, size, out);
+
         GF_OPTION_INIT ("page-count", conf->page_count, uint32, out);
 
         GF_OPTION_INIT ("force-atime-update", conf->force_atime_update, bool, out);
@@ -1075,9 +1182,7 @@ init (xlator_t *this)
 
 out:
         if (ret == -1) {
-                if (conf != NULL) {
-                        GF_FREE (conf);
-                }
+                GF_FREE (conf);
         }
 
         return ret;
@@ -1096,11 +1201,14 @@ fini (xlator_t *this)
                 goto out;
         }
 
+        this->private = NULL;
+
+        GF_ASSERT ((conf->files.next == &conf->files)
+                   && (conf->files.prev == &conf->files));
+
         pthread_mutex_destroy (&conf->conf_lock);
         GF_FREE (conf);
 
-        this->private = NULL;
-
 out:
         return;
 }
@@ -1115,6 +1223,8 @@ struct xlator_fops fops = {
         .truncate    = ra_truncate,
         .ftruncate   = ra_ftruncate,
         .fstat       = ra_fstat,
+	.discard     = ra_discard,
+        .zerofill    = ra_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -1138,5 +1248,12 @@ struct volume_options options[] = {
           .default_value = "4",
           .description = "Number of pages that will be pre-fetched"
         },
+	{ .key = {"page-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .min = 4096,
+	  .max = 1048576 * 64,
+	  .default_value = "131072",
+	  .description = "Page size with which read-ahead performs server I/O"
+	},
         { .key = {NULL} },
 };
diff --git a/xlators/performance/read-ahead/src/read-ahead.h b/xlators/performance/read-ahead/src/read-ahead.h
index f5e73cb66..d1d768c34 100644
--- a/xlators/performance/read-ahead/src/read-ahead.h
+++ b/xlators/performance/read-ahead/src/read-ahead.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __READ_AHEAD_H
@@ -85,6 +76,7 @@ struct ra_page {
         size_t            size;
         struct ra_waitq  *waitq;
         struct iobref    *iobref;
+        char              stale;
 };
 
 
diff --git a/xlators/protocol/legacy/server/Makefile.am b/xlators/performance/readdir-ahead/Makefile.am
index d471a3f92..a985f42a8 100644
--- a/xlators/protocol/legacy/server/Makefile.am
+++ b/xlators/performance/readdir-ahead/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/performance/readdir-ahead/src/Makefile.am b/xlators/performance/readdir-ahead/src/Makefile.am
new file mode 100644
index 000000000..539d6ede4
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = readdir-ahead.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
+
+readdir_ahead_la_LDFLAGS = -module -avoid-version
+
+readdir_ahead_la_SOURCES = readdir-ahead.c
+readdir_ahead_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = readdir-ahead.h readdir-ahead-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h b/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h
new file mode 100644
index 000000000..39e2c5369
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h
@@ -0,0 +1,24 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __RDA_MEM_TYPES_H__
+#define __RDA_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_rda_mem_types_ {
+        gf_rda_mt_rda_local   = gf_common_mt_end + 1,
+	gf_rda_mt_rda_fd_ctx,
+	gf_rda_mt_rda_priv,
+        gf_rda_mt_end
+};
+
+#endif
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.c b/xlators/performance/readdir-ahead/src/readdir-ahead.c
new file mode 100644
index 000000000..53e6756f0
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.c
@@ -0,0 +1,560 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/*
+ * performance/readdir-ahead preloads a local buffer with directory entries
+ * on opendir. The optimization involves using maximum sized gluster rpc
+ * requests (128k) to minimize overhead of smaller client requests.
+ *
+ * For example, fuse currently supports a maximum readdir buffer of 4k
+ * (regardless of the filesystem client's buffer size). readdir-ahead should
+ * effectively convert these smaller requests into fewer, larger sized requests
+ * for simple, sequential workloads (i.e., ls).
+ *
+ * The translator is currently designed to handle the simple, sequential case
+ * only. If a non-sequential directory read occurs, readdir-ahead disables
+ * preloads on the directory.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "call-stub.h"
+#include "readdir-ahead.h"
+#include "readdir-ahead-mem-types.h"
+#include "defaults.h"
+
+static int rda_fill_fd(call_frame_t *, xlator_t *, fd_t *);
+
+/*
+ * Get (or create) the fd context for storing prepopulated directory
+ * entries.
+ */
+static struct
+rda_fd_ctx *get_rda_fd_ctx(fd_t *fd, xlator_t *this)
+{
+	uint64_t val;
+	struct rda_fd_ctx *ctx;
+
+	LOCK(&fd->lock);
+
+	if (__fd_ctx_get(fd, this, &val) < 0) {
+		ctx = GF_CALLOC(1, sizeof(struct rda_fd_ctx),
+				gf_rda_mt_rda_fd_ctx);
+		if (!ctx)
+			goto out;
+
+		LOCK_INIT(&ctx->lock);
+		INIT_LIST_HEAD(&ctx->entries.list);
+		ctx->state = RDA_FD_NEW;
+		/* ctx offset values initialized to 0 */
+
+		if (__fd_ctx_set(fd, this, (uint64_t) ctx) < 0) {
+			GF_FREE(ctx);
+			ctx = NULL;
+			goto out;
+		}
+	} else {
+		ctx = (struct rda_fd_ctx *) val;
+	}
+out:
+	UNLOCK(&fd->lock);
+	return ctx;
+}
+
+/*
+ * Reset the tracking state of the context.
+ */
+static void
+rda_reset_ctx(struct rda_fd_ctx *ctx)
+{
+	ctx->state = RDA_FD_NEW;
+	ctx->cur_offset = 0;
+	ctx->cur_size = 0;
+	ctx->next_offset = 0;
+	gf_dirent_free(&ctx->entries);
+}
+
+/*
+ * Check whether we can handle a request. Offset verification is done by the
+ * caller, so we only check whether the preload buffer has completion status
+ * (including an error) or has some data to return.
+ */
+static gf_boolean_t
+rda_can_serve_readdirp(struct rda_fd_ctx *ctx, size_t request_size)
+{
+	if ((ctx->state & RDA_FD_EOD) ||
+	    (ctx->state & RDA_FD_ERROR) ||
+	    (!(ctx->state & RDA_FD_PLUGGED) && (ctx->cur_size > 0)))
+		return _gf_true;
+
+	return _gf_false;
+}
+
+/*
+ * Serve a request from the fd dentry list based on the size of the request
+ * buffer. ctx must be locked.
+ */
+static int32_t
+__rda_serve_readdirp(xlator_t *this, gf_dirent_t *entries, size_t request_size,
+		   struct rda_fd_ctx *ctx)
+{
+	gf_dirent_t *dirent, *tmp;
+	size_t dirent_size, size = 0;
+	int32_t count = 0;
+	struct rda_priv *priv = this->private;
+
+	list_for_each_entry_safe(dirent, tmp, &ctx->entries.list, list) {
+		dirent_size = gf_dirent_size(dirent->d_name);
+		if (size + dirent_size > request_size)
+			break;
+
+		size += dirent_size;
+		list_del_init(&dirent->list);
+		ctx->cur_size -= dirent_size;
+
+		list_add_tail(&dirent->list, &entries->list);
+		ctx->cur_offset = dirent->d_off;
+		count++;
+	}
+
+	if (ctx->cur_size <= priv->rda_low_wmark)
+		ctx->state |= RDA_FD_PLUGGED;
+
+	return count;
+}
+
+static int32_t
+rda_readdirp_stub(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+		  off_t offset, dict_t *xdata)
+{
+	gf_dirent_t entries;
+	int32_t ret;
+	struct rda_fd_ctx *ctx;
+	int op_errno = 0;
+
+	ctx = get_rda_fd_ctx(fd, this);
+	INIT_LIST_HEAD(&entries.list);
+	ret = __rda_serve_readdirp(this, &entries, size, ctx);
+
+	if (!ret && (ctx->state & RDA_FD_ERROR)) {
+		ret = -1;
+		op_errno = ctx->op_errno;
+		ctx->state &= ~RDA_FD_ERROR;
+
+		/*
+		 * the preload has stopped running in the event of an error, so
+		 * pass all future requests along
+		 */
+		ctx->state |= RDA_FD_BYPASS;
+	}
+
+	STACK_UNWIND_STRICT(readdirp, frame, ret, op_errno, &entries, xdata);
+	gf_dirent_free(&entries);
+
+	return 0;
+}
+
+static int32_t
+rda_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	     off_t off, dict_t *xdata)
+{
+	struct rda_fd_ctx *ctx;
+	call_stub_t *stub;
+	int fill = 0;
+
+	ctx = get_rda_fd_ctx(fd, this);
+	if (!ctx)
+		goto err;
+
+	if (ctx->state & RDA_FD_BYPASS)
+		goto bypass;
+
+	LOCK(&ctx->lock);
+
+	/* recheck now that we have the lock */
+	if (ctx->state & RDA_FD_BYPASS) {
+		UNLOCK(&ctx->lock);
+		goto bypass;
+	}
+
+	/*
+	 * If a new read comes in at offset 0 and the buffer has been
+	 * completed, reset the context and kickstart the filler again.
+	 */
+	if (!off && (ctx->state & RDA_FD_EOD) && (ctx->cur_size == 0)) {
+		rda_reset_ctx(ctx);
+		fill = 1;
+	}
+
+	/*
+	 * If a readdir occurs at an unexpected offset or we already have a
+	 * request pending, admit defeat and just get out of the way.
+	 */
+	if (off != ctx->cur_offset || ctx->stub) {
+		ctx->state |= RDA_FD_BYPASS;
+		UNLOCK(&ctx->lock);
+		goto bypass;
+	}
+
+	stub = fop_readdirp_stub(frame, rda_readdirp_stub, fd, size, off, xdata);
+	if (!stub) {
+		UNLOCK(&ctx->lock);
+		goto err;
+	}
+
+	/*
+	 * If we haven't bypassed the preload, this means we can either serve
+	 * the request out of the preload or the request that enables us to do
+	 * so is in flight...
+	 */
+	if (rda_can_serve_readdirp(ctx, size))
+		call_resume(stub);
+	else
+		ctx->stub = stub;
+
+	UNLOCK(&ctx->lock);
+
+	if (fill)
+		rda_fill_fd(frame, this, fd);
+
+	return 0;
+
+bypass:
+	STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+	return 0;
+
+err:
+	STACK_UNWIND_STRICT(readdirp, frame, -1, ENOMEM, NULL, NULL);
+	return 0;
+}
+
+static int32_t
+rda_fill_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		 dict_t *xdata)
+{
+	gf_dirent_t *dirent, *tmp;
+	struct rda_local *local = frame->local;
+	struct rda_fd_ctx *ctx = local->ctx;
+	struct rda_priv *priv = this->private;
+	int fill = 1;
+
+	LOCK(&ctx->lock);
+
+	/* Verify that the preload buffer is still pending on this data. */
+	if (ctx->next_offset != local->offset) {
+		gf_log(this->name, GF_LOG_ERROR,
+			"Out of sequence directory preload.");
+		ctx->state |= (RDA_FD_BYPASS|RDA_FD_ERROR);
+		ctx->op_errno = EUCLEAN;
+
+		goto out;
+	}
+
+	if (entries) {
+		list_for_each_entry_safe(dirent, tmp, &entries->list, list) {
+			list_del_init(&dirent->list);
+			/* must preserve entry order */
+			list_add_tail(&dirent->list, &ctx->entries.list);
+
+			ctx->cur_size += gf_dirent_size(dirent->d_name);
+			ctx->next_offset = dirent->d_off;
+		}
+	}
+
+	if (ctx->cur_size >= priv->rda_high_wmark)
+		ctx->state &= ~RDA_FD_PLUGGED;
+
+	if (!op_ret) {
+		/* we've hit eod */
+		ctx->state &= ~RDA_FD_RUNNING;
+		ctx->state |= RDA_FD_EOD;
+	} else if (op_ret == -1) {
+		/* kill the preload and pend the error */
+		ctx->state &= ~RDA_FD_RUNNING;
+		ctx->state |= RDA_FD_ERROR;
+		ctx->op_errno = op_errno;
+	}
+
+	/*
+	 * NOTE: The strict bypass logic in readdirp() means a pending request
+	 * is always based on ctx->cur_offset.
+	 */
+	if (ctx->stub &&
+	    rda_can_serve_readdirp(ctx, ctx->stub->args.size)) {
+		call_resume(ctx->stub);
+		ctx->stub = NULL;
+	}
+
+out:
+	/*
+	 * If we have been marked for bypass and have no pending stub, clear the
+	 * run state so we stop preloading the context with entries.
+	 */
+	if ((ctx->state & RDA_FD_BYPASS) && !ctx->stub)
+		ctx->state &= ~RDA_FD_RUNNING;
+
+	if (!(ctx->state & RDA_FD_RUNNING)) {
+		fill = 0;
+		STACK_DESTROY(ctx->fill_frame->root);
+		ctx->fill_frame = NULL;
+	}
+
+	UNLOCK(&ctx->lock);
+
+	if (fill)
+		rda_fill_fd(frame, this, local->fd);
+
+	return 0;
+}
+
+/*
+ * Start prepopulating the fd context with directory entries.
+ */
+static int
+rda_fill_fd(call_frame_t *frame, xlator_t *this, fd_t *fd)
+{
+	call_frame_t *nframe = NULL;
+	struct rda_local *local = NULL;
+	struct rda_fd_ctx *ctx;
+	off_t offset;
+	struct rda_priv *priv = this->private;
+
+	ctx = get_rda_fd_ctx(fd, this);
+	if (!ctx)
+		goto err;
+
+	LOCK(&ctx->lock);
+
+	if (ctx->state & RDA_FD_NEW) {
+		ctx->state &= ~RDA_FD_NEW;
+		ctx->state |= RDA_FD_RUNNING;
+		if (priv->rda_low_wmark)
+			ctx->state |= RDA_FD_PLUGGED;
+	}
+
+	offset = ctx->next_offset;
+
+	if (!ctx->fill_frame) {
+		nframe = copy_frame(frame);
+		if (!nframe) {
+			UNLOCK(&ctx->lock);
+			goto err;
+		}
+
+		local = mem_get0(this->local_pool);
+		if (!local) {
+			UNLOCK(&ctx->lock);
+			goto err;
+		}
+
+		local->ctx = ctx;
+		local->fd = fd;
+		nframe->local = local;
+
+		ctx->fill_frame = nframe;
+	} else {
+		nframe = ctx->fill_frame;
+		local = nframe->local;
+	}
+
+	local->offset = offset;
+
+	UNLOCK(&ctx->lock);
+
+	STACK_WIND(nframe, rda_fill_fd_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, priv->rda_req_size,
+		   offset, NULL);
+
+	return 0;
+
+err:
+	if (nframe)
+		FRAME_DESTROY(nframe);
+
+	return -1;
+}
+
+static int32_t
+rda_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+	if (!op_ret)
+		rda_fill_fd(frame, this, fd);
+
+	STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
+	return 0;
+}
+
+static int32_t
+rda_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+		dict_t *xdata)
+{
+	STACK_WIND(frame, rda_opendir_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+	return 0;
+}
+
+static int32_t
+rda_releasedir(xlator_t *this, fd_t *fd)
+{
+	uint64_t val;
+	struct rda_fd_ctx *ctx;
+
+	if (fd_ctx_del(fd, this, &val) < 0)
+		return -1;
+
+	ctx = (struct rda_fd_ctx *) val;
+	if (!ctx)
+		return 0;
+
+	rda_reset_ctx(ctx);
+
+	if (ctx->fill_frame)
+		STACK_DESTROY(ctx->fill_frame->root);
+
+	if (ctx->stub)
+		gf_log(this->name, GF_LOG_ERROR,
+			"released a directory with a pending stub");
+
+	GF_FREE(ctx);
+	return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+	int ret = -1;
+
+	if (!this)
+		goto out;
+
+	ret = xlator_mem_acct_init(this, gf_rda_mt_end + 1);
+
+	if (ret != 0)
+		gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+		       "failed");
+
+out:
+	return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+	struct rda_priv *priv = this->private;
+
+	GF_OPTION_RECONF("rda-request-size", priv->rda_req_size, options,
+			 uint32, err);
+	GF_OPTION_RECONF("rda-low-wmark", priv->rda_low_wmark, options, size,
+			 err);
+	GF_OPTION_RECONF("rda-high-wmark", priv->rda_high_wmark, options, size,
+			 err);
+
+	return 0;
+err:
+	return -1;
+}
+
+int
+init(xlator_t *this)
+{
+	struct rda_priv *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO("readdir-ahead", this, err);
+
+        if (!this->children || this->children->next) {
+                gf_log(this->name,  GF_LOG_ERROR,
+                        "FATAL: readdir-ahead not configured with exactly one"
+                        " child");
+                goto err;
+        }
+
+        if (!this->parents) {
+                gf_log(this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+	priv = GF_CALLOC(1, sizeof(struct rda_priv), gf_rda_mt_rda_priv);
+	if (!priv)
+		goto err;
+	this->private = priv;
+
+	this->local_pool = mem_pool_new(struct rda_local, 32);
+	if (!this->local_pool)
+		goto err;
+
+	GF_OPTION_INIT("rda-request-size", priv->rda_req_size, uint32, err);
+	GF_OPTION_INIT("rda-low-wmark", priv->rda_low_wmark, size, err);
+	GF_OPTION_INIT("rda-high-wmark", priv->rda_high_wmark, size, err);
+
+	return 0;
+
+err:
+	if (this->local_pool)
+		mem_pool_destroy(this->local_pool);
+	if (priv)
+		GF_FREE(priv);
+
+        return -1;
+}
+
+
+void
+fini(xlator_t *this)
+{
+        GF_VALIDATE_OR_GOTO ("readdir-ahead", this, out);
+
+	GF_FREE(this->private);
+
+out:
+        return;
+}
+
+struct xlator_fops fops = {
+	.opendir	= rda_opendir,
+	.readdirp	= rda_readdirp,
+};
+
+struct xlator_cbks cbks = {
+	.releasedir	= rda_releasedir,
+};
+
+struct volume_options options[] = {
+	{ .key = {"rda-request-size"},
+	  .type = GF_OPTION_TYPE_INT,
+	  .min = 4096,
+	  .max = 131072,
+	  .default_value = "131072",
+	  .description = "readdir-ahead request size",
+	},
+	{ .key = {"rda-low-wmark"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .min = 0,
+	  .max = 10 * GF_UNIT_MB,
+	  .default_value = "4096",
+	  .description = "the value under which we plug",
+	},
+	{ .key = {"rda-high-wmark"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .min = 0,
+	  .max = 100 * GF_UNIT_MB,
+	  .default_value = "131072",
+	  .description = "the value over which we unplug",
+	},
+        { .key = {NULL} },
+};
+
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.h b/xlators/performance/readdir-ahead/src/readdir-ahead.h
new file mode 100644
index 000000000..e48786dae
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.h
@@ -0,0 +1,46 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __READDIR_AHEAD_H
+#define __READDIR_AHEAD_H
+
+/* state flags */
+#define RDA_FD_NEW	(1 << 0)
+#define RDA_FD_RUNNING	(1 << 1)
+#define RDA_FD_EOD	(1 << 2)
+#define RDA_FD_ERROR	(1 << 3)
+#define RDA_FD_BYPASS	(1 << 4)
+#define RDA_FD_PLUGGED	(1 << 5)
+
+struct rda_fd_ctx {
+	off_t cur_offset;	/* current head of the ctx */
+	size_t cur_size;	/* current size of the preload */
+	off_t next_offset;	/* tail of the ctx */
+	uint32_t state;
+	gf_lock_t lock;
+	gf_dirent_t entries;
+	call_frame_t *fill_frame;
+	call_stub_t *stub;
+	int op_errno;
+};
+
+struct rda_local {
+	struct rda_fd_ctx *ctx;
+	fd_t *fd;
+	off_t offset;
+};
+
+struct rda_priv {
+	uint32_t rda_req_size;
+	uint64_t rda_low_wmark;
+	uint64_t rda_high_wmark;
+};
+
+#endif /* __READDIR_AHEAD_H */
diff --git a/xlators/performance/symlink-cache/src/Makefile.am b/xlators/performance/symlink-cache/src/Makefile.am
index 06e85fc92..4091c3293 100644
--- a/xlators/performance/symlink-cache/src/Makefile.am
+++ b/xlators/performance/symlink-cache/src/Makefile.am
@@ -1,12 +1,13 @@
 xlator_LTLIBRARIES = symlink-cache.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/performance
 
-symlink_cache_la_LDFLAGS = -module -avoidversion 
+symlink_cache_la_LDFLAGS = -module -avoid-version 
 
 symlink_cache_la_SOURCES = symlink-cache.c
 symlink_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/symlink-cache/src/symlink-cache.c b/xlators/performance/symlink-cache/src/symlink-cache.c
index a82786cce..3b5fbc252 100644
--- a/xlators/performance/symlink-cache/src/symlink-cache.c
+++ b/xlators/performance/symlink-cache/src/symlink-cache.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -146,8 +137,7 @@ sc_cache_set (xlator_t *this, inode_t *inode, struct iatt *buf,
 err:
 
 	if (sc) {
-		if (sc->readlink)
-			FREE (sc->readlink);
+		FREE (sc->readlink);
 		sc->readlink = NULL;
 		FREE (sc);
 	}
@@ -242,7 +232,7 @@ sc_cache_get (xlator_t *this, inode_t *inode, char **link)
 int
 sc_readlink_cbk (call_frame_t *frame, void *cookie,
 		 xlator_t *this, int op_ret, int op_errno,
-		 const char *link, struct iatt *sbuf)
+		 const char *link, struct iatt *sbuf, dict_t *xdata)
 {
 	if (op_ret > 0)
 		sc_cache_update (this, frame->local, link);
@@ -250,14 +240,15 @@ sc_readlink_cbk (call_frame_t *frame, void *cookie,
 	inode_unref (frame->local);
 	frame->local = NULL;
 
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, link, sbuf);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, link, sbuf,
+                             xdata);
         return 0;
 }
 
 
 int
 sc_readlink (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, size_t size)
+	     loc_t *loc, size_t size, dict_t *xdata)
 {
 	char *link = NULL;
         struct iatt buf = {0, };
@@ -275,7 +266,8 @@ sc_readlink (call_frame_t *frame, xlator_t *this,
                   using buf in readlink_cbk should be aware that @buf
                   is 0 filled
                 */
-		STACK_UNWIND_STRICT (readlink, frame, strlen (link), 0, link, &buf);
+		STACK_UNWIND_STRICT (readlink, frame, strlen (link), 0, link,
+                                     &buf, NULL);
 		FREE (link);
 		return 0;
 	}
@@ -285,7 +277,7 @@ sc_readlink (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, sc_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
 
 	return 0;
 }
@@ -295,7 +287,7 @@ int
 sc_symlink_cbk (call_frame_t *frame, void *cookie,
 		xlator_t *this, int op_ret, int op_errno,
                 inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
 	if (op_ret == 0) {
 		if (frame->local) {
@@ -303,22 +295,22 @@ sc_symlink_cbk (call_frame_t *frame, void *cookie,
 		}
 	}
 
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf, preparent,
-                      postparent);
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 sc_symlink (call_frame_t *frame, xlator_t *this,
-	    const char *dst, loc_t *src, dict_t *params)
+	    const char *dst, loc_t *src, mode_t umask, dict_t *xdata)
 {
 	frame->local = strdup (dst);
 
         STACK_WIND (frame, sc_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
-                    dst, src, params);
+                    dst, src, umask, xdata);
 
 	return 0;
 }
@@ -327,7 +319,7 @@ sc_symlink (call_frame_t *frame, xlator_t *this,
 int
 sc_lookup_cbk (call_frame_t *frame, void *cookie,
 	       xlator_t *this, int op_ret, int op_errno,
-	       inode_t *inode, struct iatt *buf, dict_t *xattr,
+	       inode_t *inode, struct iatt *buf, dict_t *xdata,
                struct iatt *postparent)
 {
 	if (op_ret == 0)
@@ -335,19 +327,20 @@ sc_lookup_cbk (call_frame_t *frame, void *cookie,
 	else
 		sc_cache_flush (this, inode);
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xattr, postparent);
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
         return 0;
 }
 
 
 int
 sc_lookup (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, dict_t *xattr_req)
+	   loc_t *loc, dict_t *xdata)
 {
         STACK_WIND (frame, sc_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
+                    loc, xdata);
 
         return 0;
 }
@@ -363,10 +356,9 @@ sc_forget (xlator_t *this,
 }
 
 
-int32_t 
+int32_t
 init (xlator_t *this)
 {
-	
         if (!this->children || this->children->next)
         {
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/performance/write-behind/src/Makefile.am b/xlators/performance/write-behind/src/Makefile.am
index a5ebc90bd..6c829d8ee 100644
--- a/xlators/performance/write-behind/src/Makefile.am
+++ b/xlators/performance/write-behind/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = write-behind.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-write_behind_la_LDFLAGS = -module -avoidversion 
+write_behind_la_LDFLAGS = -module -avoid-version 
 
 write_behind_la_SOURCES = write-behind.c
 write_behind_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = write-behind-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/performance/write-behind/src/write-behind-mem-types.h b/xlators/performance/write-behind/src/write-behind-mem-types.h
index 0b03875fd..f64f429ce 100644
--- a/xlators/performance/write-behind/src/write-behind-mem-types.h
+++ b/xlators/performance/write-behind/src/write-behind-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -28,6 +19,7 @@ enum gf_wb_mem_types_ {
         gf_wb_mt_wb_request_t,
         gf_wb_mt_iovec,
         gf_wb_mt_wb_conf_t,
+        gf_wb_mt_wb_inode_t,
         gf_wb_mt_end
 };
 #endif
diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c
index cc0569dd4..00457338d 100644
--- a/xlators/performance/write-behind/src/write-behind.c
+++ b/xlators/performance/write-behind/src/write-behind.c
@@ -1,24 +1,13 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-/*TODO: check for non null wb_file_data before getting wb_file */
-
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -35,2669 +24,1932 @@
 #include "common-utils.h"
 #include "call-stub.h"
 #include "statedump.h"
+#include "defaults.h"
 #include "write-behind-mem-types.h"
 
-#define MAX_VECTOR_COUNT  8
-#define WB_AGGREGATE_SIZE 131072 /* 128 KB */
-#define WB_WINDOW_SIZE    1048576 /* 1MB */
+#define MAX_VECTOR_COUNT          8
+#define WB_AGGREGATE_SIZE         131072 /* 128 KB */
+#define WB_WINDOW_SIZE            1048576 /* 1MB */
 
 typedef struct list_head list_head_t;
 struct wb_conf;
-struct wb_page;
-struct wb_file;
-
-typedef struct wb_file {
-        int          disabled;
-        uint64_t     disable_till;
-        size_t       window_conf;
-        size_t       window_current;
-        int32_t      flags;
-        size_t       aggregate_current;
-        int32_t      refcount;
-        int32_t      op_ret;
-        int32_t      op_errno;
-        list_head_t  request;
-        list_head_t  passive_requests;
-        fd_t        *fd;
+struct wb_inode;
+
+typedef struct wb_inode {
+        ssize_t      window_conf;
+        ssize_t      window_current;
+	ssize_t      transit; /* size of data stack_wound, and yet
+				 to be fulfilled (wb_fulfill_cbk).
+				 used for trickling_writes
+			      */
+
+        list_head_t  all;    /* All requests, from enqueue() till destroy().
+				Used only for resetting generation
+				number when empty.
+			     */
+        list_head_t  todo;   /* Work to do (i.e, STACK_WIND to server).
+				Once we STACK_WIND, the entry is taken
+				off the list. If it is non-sync write,
+				then we continue to track it via @liability
+				or @temptation depending on the status
+				of its writeback.
+			     */
+        list_head_t  liability;   /* Non-sync writes which are lied
+				     (STACK_UNWIND'ed to caller) but ack
+				     from server not yet complete. This
+				     is the "liability" which we hold, and
+				     must guarantee that dependent operations
+				     which arrive later (which overlap, etc.)
+				     are issued only after their dependencies
+				     in this list are "fulfilled".
+
+				     Server acks for entries in this list
+				     shrinks the window.
+
+				     The sum total of all req->write_size
+				     of entries in this list must be kept less
+				     than the permitted window size.
+				  */
+        list_head_t  temptation;  /* Operations for which we are tempted
+				     to 'lie' (write-behind), but temporarily
+				     holding off (because of insufficient
+				     window capacity, etc.)
+
+				     This is the list to look at to grow
+				     the window (in __wb_pick_unwinds()).
+
+				     Entries typically get chosen from
+				     write-behind from this list, and therefore
+				     get "upgraded" to the "liability" list.
+			     */
+	list_head_t  wip; /* List of write calls in progress, SYNC or non-SYNC
+			     which are currently STACK_WIND'ed towards the server.
+			     This is for guaranteeing that no two overlapping
+			     writes are in progress at the same time. Modules
+			     like eager-lock in AFR depend on this behavior.
+			  */
+	uint64_t     gen;    /* Liability generation number. Represents
+				the current 'state' of liability. Every
+				new addition to the liability list bumps
+				the generation number.
+
+				a newly arrived request is only required
+				to perform causal checks against the entries
+				in the liability list which were present
+				at the time of its addition. the generation
+				number at the time of its addition is stored
+				in the request and used during checks.
+
+				the liability list can grow while the request
+				waits in the todo list waiting for its
+				dependent operations to complete. however
+				it is not of the request's concern to depend
+				itself on those new entries which arrived
+				after it arrived (i.e, those that have a
+				liability generation higher than itself)
+			     */
+	size_t       size; /* Size of the file to catch write after EOF. */
         gf_lock_t    lock;
         xlator_t    *this;
-}wb_file_t;
+} wb_inode_t;
+
 
 typedef struct wb_request {
-        list_head_t     list;
-        list_head_t     winds;
-        list_head_t     unwinds;
-        list_head_t     other_requests;
-        call_stub_t    *stub;
-        size_t          write_size;
-        int32_t         refcount;
-        wb_file_t      *file;
-        glusterfs_fop_t fop;
-        union {
-                struct  {
-                        char write_behind;
-                        char stack_wound;
-                        char got_reply;
-                        char virgin;
-                        char flush_all;     /* while trying to sync to back-end,
-                                             * don't wait till a data of size
-                                             * equal to configured aggregate-size
-                                             * is accumulated, instead sync
-                                             * whatever data currently present in
-                                             * request queue.
-                                             */
-
-                }write_request;
-
-                struct {
-                        char marked_for_resume;
-                }other_requests;
-        }flags;
+        list_head_t           all;
+        list_head_t           todo;
+	list_head_t           lie;  /* either in @liability or @temptation */
+        list_head_t           winds;
+        list_head_t           unwinds;
+        list_head_t           wip;
+
+        call_stub_t          *stub;
+
+        ssize_t               write_size;  /* currently held size
+					      (after collapsing) */
+	size_t                orig_size;   /* size which arrived with the request.
+					      This is the size by which we grow
+					      the window when unwinding the frame.
+					   */
+        size_t                total_size;  /* valid only in @head in wb_fulfill().
+					      This is the size with which we perform
+					      STACK_WIND to server and therefore the
+					      amount by which we shrink the window.
+					   */
+
+	int                   op_ret;
+	int                   op_errno;
+
+        int32_t               refcount;
+        wb_inode_t           *wb_inode;
+        glusterfs_fop_t       fop;
+        gf_lkowner_t          lk_owner;
+	struct iobref        *iobref;
+	uint64_t              gen;  /* inode liability state at the time of
+				       request arrival */
+
+	fd_t                 *fd;
+	struct {
+		size_t        size;          /* 0 size == till infinity */
+		off_t         off;
+		int           append:1;      /* offset is invalid. only one
+						outstanding append at a time */
+		int           tempted:1;     /* true only for non-sync writes */
+		int           lied:1;        /* sin committed */
+		int           fulfilled:1;   /* got server acknowledgement */
+		int           go:1;          /* enough aggregating, good to go */
+	} ordering;
 } wb_request_t;
 
-struct wb_conf {
-        uint64_t     aggregate_size;
-        uint64_t     window_size;
-        uint64_t     disable_till;
-        gf_boolean_t enable_O_SYNC;
-        gf_boolean_t flush_behind;
-        gf_boolean_t enable_trickling_writes;
-};
-
-typedef struct wb_local {
-        list_head_t     winds;
-        int32_t         flags;
-        int32_t         wbflags;
-        struct wb_file *file;
-        wb_request_t   *request;
-        int             op_ret;
-        int             op_errno;
-        call_frame_t   *frame;
-        int32_t         reply_count;
-} wb_local_t;
-
-typedef struct wb_conf wb_conf_t;
-typedef struct wb_page wb_page_t;
 
-int32_t
-wb_process_queue (call_frame_t *frame, wb_file_t *file);
+typedef struct wb_conf {
+        uint64_t         aggregate_size;
+        uint64_t         window_size;
+        gf_boolean_t     flush_behind;
+        gf_boolean_t     trickling_writes;
+	gf_boolean_t     strict_write_ordering;
+	gf_boolean_t     strict_O_DIRECT;
+} wb_conf_t;
 
-ssize_t
-wb_sync (call_frame_t *frame, wb_file_t *file, list_head_t *winds);
 
-ssize_t
-__wb_mark_winds (list_head_t *list, list_head_t *winds, size_t aggregate_size,
-                 char enable_trickling_writes);
+void
+wb_process_queue (wb_inode_t *wb_inode);
 
 
-static int
-__wb_request_unref (wb_request_t *this)
+wb_inode_t *
+__wb_inode_ctx_get (xlator_t *this, inode_t *inode)
 {
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
-
-        if (this->refcount <= 0) {
-                gf_log ("wb-request", GF_LOG_WARNING,
-                        "refcount(%d) is <= 0", this->refcount);
-                goto out;
-        }
+        uint64_t    value    = 0;
+        wb_inode_t *wb_inode = NULL;
 
-        ret = --this->refcount;
-        if (this->refcount == 0) {
-                list_del_init (&this->list);
-                if (this->stub && this->stub->fop == GF_FOP_WRITE) {
-                        call_stub_destroy (this->stub);
-                }
+        __inode_ctx_get (inode, this, &value);
+        wb_inode = (wb_inode_t *)(unsigned long) value;
 
-                GF_FREE (this);
-        }
-
-out:
-        return ret;
+        return wb_inode;
 }
 
 
-static int
-wb_request_unref (wb_request_t *this)
+wb_inode_t *
+wb_inode_ctx_get (xlator_t *this, inode_t *inode)
 {
-        wb_file_t *file = NULL;
-        int        ret  = -1;
+        wb_inode_t *wb_inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("write-behind", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
-        file = this->file;
-
-        LOCK (&file->lock);
+        LOCK (&inode->lock);
         {
-                ret = __wb_request_unref (this);
+                wb_inode = __wb_inode_ctx_get (this, inode);
         }
-        UNLOCK (&file->lock);
-
+        UNLOCK (&inode->lock);
 out:
-        return ret;
+        return wb_inode;
 }
 
 
-static wb_request_t *
-__wb_request_ref (wb_request_t *this)
+gf_boolean_t
+wb_fd_err (fd_t *fd, xlator_t *this, int32_t *op_errno)
 {
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
-
-        if (this->refcount < 0) {
-                gf_log ("wb-request", GF_LOG_WARNING,
-                        "refcount(%d) is < 0", this->refcount);
-                this = NULL;
-                goto out;
-        }
-
-        this->refcount++;
-
-out:
-        return this;
-}
-
-
-wb_request_t *
-wb_request_ref (wb_request_t *this)
-{
-        wb_file_t *file = NULL;
+        gf_boolean_t err   = _gf_false;
+        uint64_t     value = 0;
+        int32_t      tmp   = 0;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
+	if (fd_ctx_get (fd, this, &value) == 0) {
+                if (op_errno) {
+                        tmp = value;
+                        *op_errno = tmp;
+                }
 
-        file = this->file;
-        LOCK (&file->lock);
-        {
-                this = __wb_request_ref (this);
+                err = _gf_true;
         }
-        UNLOCK (&file->lock);
 
-out:
-        return this;
+        return err;
 }
 
 
-wb_request_t *
-wb_enqueue (wb_file_t *file, call_stub_t *stub)
-{
-        wb_request_t *request = NULL, *tmp = NULL;
-        call_frame_t *frame   = NULL;
-        wb_local_t   *local   = NULL;
-        struct iovec *vector  = NULL;
-        int32_t       count   = 0;
+/*
+  Below is a succinct explanation of the code deciding whether two regions
+  overlap, from Pavan <tcp@gluster.com>.
 
-        GF_VALIDATE_OR_GOTO ("write-behind", file, out);
-        GF_VALIDATE_OR_GOTO (file->this->name, stub, out);
+  For any two ranges to be non-overlapping, either the end of the first
+  range is lesser than the start of the second, or vice versa. Example -
 
-        request = GF_CALLOC (1, sizeof (*request), gf_wb_mt_wb_request_t);
-        if (request == NULL) {
-                goto out;
-        }
+  <--------->       <-------------->
+  p         q       x              y
 
-        INIT_LIST_HEAD (&request->list);
-        INIT_LIST_HEAD (&request->winds);
-        INIT_LIST_HEAD (&request->unwinds);
-        INIT_LIST_HEAD (&request->other_requests);
+  ( q < x ) or (y < p) = > No overlap.
 
-        request->stub = stub;
-        request->file = file;
-        request->fop  = stub->fop;
+  To check for *overlap*, we can negate this (using de morgan's laws), and
+  it becomes -
 
-        frame = stub->frame;
-        local = frame->local;
-        if (local) {
-                local->request = request;
-        }
+  (q >= x ) and (y >= p)
 
-        if (stub->fop == GF_FOP_WRITE) {
-                vector = stub->args.writev.vector;
-                count = stub->args.writev.count;
+  Either that, or you write the negation using -
 
-                request->write_size = iov_length (vector, count);
-                if (local) {
-                        local->op_ret = request->write_size;
-                        local->op_errno = 0;
-                }
-
-                request->flags.write_request.virgin = 1;
-        }
-
-        LOCK (&file->lock);
-        {
-                list_add_tail (&request->list, &file->request);
-                if (stub->fop == GF_FOP_WRITE) {
-                        /* reference for stack winding */
-                        __wb_request_ref (request);
-
-                        /* reference for stack unwinding */
-                        __wb_request_ref (request);
-
-                        file->aggregate_current += request->write_size;
-                } else {
-                        list_for_each_entry (tmp, &file->request, list) {
-                                if (tmp->stub && tmp->stub->fop
-                                    == GF_FOP_WRITE) {
-                                        tmp->flags.write_request.flush_all = 1;
-                                }
-                        }
-
-                        /*reference for resuming */
-                        __wb_request_ref (request);
-                }
-        }
-        UNLOCK (&file->lock);
+  if (! ((q < x) or (y < p)) ) {
+  "Overlap"
+  }
+*/
 
-out:
-        return request;
+gf_boolean_t
+wb_requests_overlap (wb_request_t *req1, wb_request_t *req2)
+{
+        uint64_t         r1_start = 0;
+	uint64_t         r1_end = 0;
+	uint64_t         r2_start = 0;
+	uint64_t         r2_end = 0;
+        enum _gf_boolean do_overlap = 0;
+
+        r1_start = req1->ordering.off;
+	if (req1->ordering.size)
+		r1_end = r1_start + req1->ordering.size - 1;
+	else
+		r1_end = ULLONG_MAX;
+
+        r2_start = req2->ordering.off;
+	if (req2->ordering.size)
+		r2_end = r2_start + req2->ordering.size - 1;
+	else
+		r2_end = ULLONG_MAX;
+
+        do_overlap = ((r1_end >= r2_start) && (r2_end >= r1_start));
+
+        return do_overlap;
 }
 
 
-wb_file_t *
-wb_file_create (xlator_t *this, fd_t *fd, int32_t flags)
+gf_boolean_t
+wb_requests_conflict (wb_request_t *lie, wb_request_t *req)
 {
-        wb_file_t *file = NULL;
-        wb_conf_t *conf = NULL;
+	wb_conf_t  *conf = NULL;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+	conf = req->wb_inode->this->private;
 
-        conf = this->private;
+	if (lie == req)
+		/* request cannot conflict with itself */
+		return _gf_false;
 
-        file = GF_CALLOC (1, sizeof (*file), gf_wb_mt_wb_file_t);
-        if (file == NULL) {
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&file->request);
-        INIT_LIST_HEAD (&file->passive_requests);
-
-        /*
-          fd_ref() not required, file should never decide the existence of
-          an fd
-        */
-        file->fd= fd;
-        file->disable_till = conf->disable_till;
-        file->this = this;
-        file->refcount = 1;
-        file->window_conf = conf->window_size;
-        file->flags = flags;
+	if (lie->gen >= req->gen)
+		/* this liability entry was behind
+		   us in the todo list */
+		return _gf_false;
 
-        LOCK_INIT (&file->lock);
+	if (lie->ordering.append)
+		/* all modifications wait for the completion
+		   of outstanding append */
+		return _gf_true;
 
-        fd_ctx_set (fd, this, (uint64_t)(long)file);
+	if (conf->strict_write_ordering)
+		/* We are sure (lie->gen < req->gen) by now. So
+		   skip overlap check if strict write ordering is
+		   requested and always return "conflict" against a
+		   lower generation lie. */
+		return _gf_true;
 
-out:
-        return file;
+	return wb_requests_overlap (lie, req);
 }
 
 
-void
-wb_file_destroy (wb_file_t *file)
+gf_boolean_t
+wb_liability_has_conflict (wb_inode_t *wb_inode, wb_request_t *req)
 {
-        int32_t refcount = 0;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", file, out);
+        wb_request_t *each     = NULL;
 
-        LOCK (&file->lock);
-        {
-                refcount = --file->refcount;
-        }
-        UNLOCK (&file->lock);
-
-        if (!refcount){
-                LOCK_DESTROY (&file->lock);
-                GF_FREE (file);
+        list_for_each_entry (each, &wb_inode->liability, lie) {
+		if (wb_requests_conflict (each, req))
+			return _gf_true;
         }
 
-out:
-        return;
+        return _gf_false;
 }
 
 
-int32_t
-wb_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf)
+gf_boolean_t
+wb_wip_has_conflict (wb_inode_t *wb_inode, wb_request_t *req)
 {
-        wb_local_t   *local             = NULL;
-        list_head_t  *winds             = NULL;
-        wb_file_t    *file              = NULL;
-        wb_request_t *request           = NULL, *dummy = NULL;
-        wb_local_t   *per_request_local = NULL;
-        int32_t       ret               = -1;
-        fd_t         *fd                = NULL;
+        wb_request_t *each     = NULL;
 
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+	if (req->stub->fop != GF_FOP_WRITE)
+		/* non-writes fundamentally never conflict with WIP requests */
+		return _gf_false;
 
-        local = frame->local;
-        winds = &local->winds;
+        list_for_each_entry (each, &wb_inode->wip, wip) {
+		if (each == req)
+			/* request never conflicts with itself,
+			   though this condition should never occur.
+			*/
+			continue;
 
-        file = local->file;
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
-
-        LOCK (&file->lock);
-        {
-                list_for_each_entry_safe (request, dummy, winds, winds) {
-                        request->flags.write_request.got_reply = 1;
-
-                        if (!request->flags.write_request.write_behind
-                            && (op_ret == -1)) {
-                                per_request_local = request->stub->frame->local;
-                                per_request_local->op_ret = op_ret;
-                                per_request_local->op_errno = op_errno;
-                        }
-
-                        if (request->flags.write_request.write_behind) {
-                                file->window_current -= request->write_size;
-                        }
-
-                        __wb_request_unref (request);
-                }
-
-                if (op_ret == -1) {
-                        file->op_ret = op_ret;
-                        file->op_errno = op_errno;
-                }
-                fd = file->fd;
-        }
-        UNLOCK (&file->lock);
-
-        ret = wb_process_queue (frame, file);
-        if (ret == -1) {
-                if (errno == ENOMEM) {
-                        LOCK (&file->lock);
-                        {
-                                file->op_ret = -1;
-                                file->op_errno = ENOMEM;
-                        }
-                        UNLOCK (&file->lock);
-                }
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "request queue processing failed");
+		if (wb_requests_overlap (each, req))
+			return _gf_true;
         }
 
-        /* safe place to do fd_unref */
-        fd_unref (fd);
-
-        STACK_DESTROY (frame->root);
-
-out:
-        return 0;
+        return _gf_false;
 }
 
 
-ssize_t
-wb_sync (call_frame_t *frame, wb_file_t *file, list_head_t *winds)
-{
-        wb_request_t   *dummy         = NULL, *request = NULL;
-        wb_request_t   *first_request = NULL, *next = NULL;
-        size_t          total_count   = 0, count = 0;
-        size_t          copied        = 0;
-        call_frame_t   *sync_frame    = NULL;
-        struct iobref  *iobref        = NULL;
-        wb_local_t     *local         = NULL;
-        struct iovec   *vector        = NULL;
-        ssize_t         current_size  = 0, bytes = 0;
-        size_t          bytecount     = 0;
-        wb_conf_t      *conf          = NULL;
-        fd_t           *fd            = NULL;
-        int32_t         op_errno      = -1;
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ((file ? file->this->name
-                                         : "write-behind"), frame,
-                                        out, bytes, -1);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, file, out, bytes,
-                                        -1);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, winds, out, bytes,
-                                        -1);
-
-        conf = file->this->private;
-        list_for_each_entry (request, winds, winds) {
-                total_count += request->stub->args.writev.count;
-                if (total_count > 0) {
-                        break;
-                }
-        }
+static int
+__wb_request_unref (wb_request_t *req)
+{
+        int         ret = -1;
+	wb_inode_t *wb_inode = NULL;
+
+	wb_inode = req->wb_inode;
 
-        if (total_count == 0) {
-                gf_log (file->this->name, GF_LOG_TRACE, "no vectors are to be"
-                        "synced");
+        if (req->refcount <= 0) {
+                gf_log ("wb-request", GF_LOG_WARNING,
+                        "refcount(%d) is <= 0", req->refcount);
                 goto out;
         }
 
-        list_for_each_entry_safe (request, dummy, winds, winds) {
-                if (!vector) {
-                        vector = GF_MALLOC (VECTORSIZE (MAX_VECTOR_COUNT),
-                                            gf_wb_mt_iovec);
-                        if (vector == NULL) {
-                                bytes = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        iobref = iobref_new ();
-                        if (iobref == NULL) {
-                                bytes = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        local = mem_get0 (THIS->local_pool);
-                        if (local == NULL) {
-                                bytes = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        INIT_LIST_HEAD (&local->winds);
-
-                        first_request = request;
-                        current_size = 0;
-                }
+        ret = --req->refcount;
+        if (req->refcount == 0) {
+                list_del_init (&req->todo);
+                list_del_init (&req->lie);
+		list_del_init (&req->wip);
 
-                count += request->stub->args.writev.count;
-                bytecount = VECTORSIZE (request->stub->args.writev.count);
-                memcpy (((char *)vector)+copied,
-                        request->stub->args.writev.vector,
-                        bytecount);
-                copied += bytecount;
+		list_del_init (&req->all);
+		if (list_empty (&wb_inode->all)) {
+			wb_inode->gen = 0;
+			/* in case of accounting errors? */
+			wb_inode->window_current = 0;
+		}
 
-                current_size += request->write_size;
+		list_del_init (&req->winds);
+		list_del_init (&req->unwinds);
 
-                if (request->stub->args.writev.iobref) {
-                        iobref_merge (iobref,
-                                      request->stub->args.writev.iobref);
-                }
+                if (req->stub && req->ordering.tempted) {
+                        call_stub_destroy (req->stub);
+			req->stub = NULL;
+                } /* else we would have call_resume()'ed */
 
-                next = NULL;
-                if (request->winds.next != winds) {
-                        next = list_entry (request->winds.next,
-                                           wb_request_t, winds);
-                }
+		if (req->iobref)
+			iobref_unref (req->iobref);
 
-                list_del_init (&request->winds);
-                list_add_tail (&request->winds, &local->winds);
-
-                if ((!next)
-                    || ((count + next->stub->args.writev.count)
-                        > MAX_VECTOR_COUNT)
-                    || ((current_size + next->write_size)
-                        > conf->aggregate_size)) {
-
-                        sync_frame = copy_frame (frame);
-                        if (sync_frame == NULL) {
-                                bytes = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        sync_frame->local = local;
-                        local->file = file;
-
-                        LOCK (&file->lock);
-                        {
-                                fd = file->fd;
-                        }
-                        UNLOCK (&file->lock);
-
-                        fd_ref (fd);
-
-                        bytes += current_size;
-                        STACK_WIND (sync_frame, wb_sync_cbk,
-                                    FIRST_CHILD(sync_frame->this),
-                                    FIRST_CHILD(sync_frame->this)->fops->writev,
-                                    fd, vector, count,
-                                    first_request->stub->args.writev.off,
-                                    first_request->stub->args.writev.flags,
-                                    iobref);
+		if (req->fd)
+			fd_unref (req->fd);
 
-                        iobref_unref (iobref);
-                        GF_FREE (vector);
-                        first_request = NULL;
-                        iobref = NULL;
-                        vector = NULL;
-                        sync_frame = NULL;
-                        local = NULL;
-                        copied = count = 0;
-                }
+                GF_FREE (req);
         }
-
 out:
-        if (sync_frame != NULL) {
-                sync_frame->local = NULL;
-                STACK_DESTROY (sync_frame->root);
-        }
-
-        if (local != NULL) {
-                /* had we winded these requests, we would have unrefed
-                 * in wb_sync_cbk.
-                 */
-                list_for_each_entry_safe (request, dummy, &local->winds,
-                                          winds) {
-                        wb_request_unref (request);
-                }
+        return ret;
+}
 
-                mem_put (local);
-                local = NULL;
-        }
 
-        if (iobref != NULL) {
-                iobref_unref (iobref);
-        }
+static int
+wb_request_unref (wb_request_t *req)
+{
+        wb_inode_t *wb_inode = NULL;
+        int         ret      = -1;
 
-        if (vector != NULL) {
-                GF_FREE (vector);
-        }
+        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
 
-        if (bytes == -1) {
-                /*
-                 * had we winded these requests, we would have unrefed
-                 * in wb_sync_cbk.
-                 */
-                if (local) {
-                        list_for_each_entry_safe (request, dummy, &local->winds,
-                                                  winds) {
-                                wb_request_unref (request);
-                        }
-                }
+        wb_inode = req->wb_inode;
 
-                if (file != NULL) {
-                        LOCK (&file->lock);
-                        {
-                                file->op_ret = -1;
-                                file->op_errno = op_errno;
-                        }
-                        UNLOCK (&file->lock);
-                }
+        LOCK (&wb_inode->lock);
+        {
+                ret = __wb_request_unref (req);
         }
+        UNLOCK (&wb_inode->lock);
 
-        return bytes;
+out:
+        return ret;
 }
 
 
-int32_t
-wb_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, struct iatt *buf)
-{
-        wb_local_t   *local         = NULL;
-        wb_request_t *request       = NULL;
-        call_frame_t *process_frame = NULL;
-        wb_file_t    *file          = NULL;
-        int32_t       ret           = -1;
-        fd_t         *fd            = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-
-        local = frame->local;
-        file = local->file;
-
-        request = local->request;
-        if (request) {
-                process_frame = copy_frame (frame);
-                if (process_frame == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
+static wb_request_t *
+__wb_request_ref (wb_request_t *req)
+{
+        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
+
+        if (req->refcount < 0) {
+                gf_log ("wb-request", GF_LOG_WARNING,
+                        "refcount(%d) is < 0", req->refcount);
+                req = NULL;
+                goto out;
         }
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        req->refcount++;
 
-        if (request != NULL) {
-                wb_request_unref (request);
-        }
+out:
+        return req;
+}
 
-        if (process_frame != NULL) {
-                ret = wb_process_queue (process_frame, file);
-                if (ret == -1) {
-                        if ((errno == ENOMEM) && (file != NULL)) {
-                                LOCK (&file->lock);
-                                {
-                                        file->op_ret = -1;
-                                        file->op_errno = ENOMEM;
-                                }
-                                UNLOCK (&file->lock);
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
 
-                STACK_DESTROY (process_frame->root);
-        }
+wb_request_t *
+wb_request_ref (wb_request_t *req)
+{
+        wb_inode_t *wb_inode = NULL;
 
-        if (file) {
-                LOCK (&file->lock);
-                {
-                        fd = file->fd;
-                }
-                UNLOCK (&file->lock);
+        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
 
-                fd_unref (fd);
+        wb_inode = req->wb_inode;
+        LOCK (&wb_inode->lock);
+        {
+                req = __wb_request_ref (req);
         }
+        UNLOCK (&wb_inode->lock);
 
-        return 0;
+out:
+        return req;
 }
 
 
-static int32_t
-wb_stat_helper (call_frame_t *frame, xlator_t *this, loc_t *loc)
+gf_boolean_t
+wb_enqueue_common (wb_inode_t *wb_inode, call_stub_t *stub, int tempted)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-
-        STACK_WIND (frame, wb_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc);
-        return 0;
-}
-
+        wb_request_t *req     = NULL;
 
-int32_t
-wb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        wb_file_t    *file     = NULL;
-        fd_t         *iter_fd  = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        if (loc->inode) {
-                /* FIXME: fd_lookup extends life of fd till stat returns */
-                iter_fd = fd_lookup (loc->inode, frame->root->pid);
-                if (iter_fd) {
-                        if (!fd_ctx_get (iter_fd, this, &tmp_file)) {
-                                file = (wb_file_t *)(long)tmp_file;
-                        } else {
-                                fd_unref (iter_fd);
-                                iter_fd = NULL;
-                        }
-                }
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+        GF_VALIDATE_OR_GOTO ("write-behind", wb_inode, out);
+        GF_VALIDATE_OR_GOTO (wb_inode->this->name, stub, out);
 
-        local->file = file;
+        req = GF_CALLOC (1, sizeof (*req), gf_wb_mt_wb_request_t);
+        if (!req)
+                goto out;
 
-        frame->local = local;
+        INIT_LIST_HEAD (&req->all);
+        INIT_LIST_HEAD (&req->todo);
+        INIT_LIST_HEAD (&req->lie);
+        INIT_LIST_HEAD (&req->winds);
+        INIT_LIST_HEAD (&req->unwinds);
+        INIT_LIST_HEAD (&req->wip);
 
-        if (file) {
-                stub = fop_stat_stub (frame, wb_stat_helper, loc);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        req->stub = stub;
+        req->wb_inode = wb_inode;
+        req->fop  = stub->fop;
+	req->ordering.tempted = tempted;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        if (stub->fop == GF_FOP_WRITE) {
+                req->write_size = iov_length (stub->args.vector,
+					      stub->args.count);
+
+		/* req->write_size can change as we collapse
+		   small writes. But the window needs to grow
+		   only by how much we acknowledge the app. so
+		   copy the original size in orig_size for the
+		   purpose of accounting.
+		*/
+		req->orig_size = req->write_size;
+
+		/* Let's be optimistic that we can
+		   lie about it
+		*/
+		req->op_ret = req->write_size;
+		req->op_errno = 0;
+
+		if (stub->args.fd->flags & O_APPEND)
+			req->ordering.append = 1;
+        }
+
+        req->lk_owner = stub->frame->root->lk_owner;
+
+	switch (stub->fop) {
+	case GF_FOP_WRITE:
+		LOCK (&wb_inode->lock);
+		{
+			if (wb_inode->size < stub->args.offset) {
+				req->ordering.off = wb_inode->size;
+				req->ordering.size = stub->args.offset
+				                     + req->write_size
+						     - wb_inode->size;
+			} else {
+				req->ordering.off = stub->args.offset;
+				req->ordering.size = req->write_size;
+			}
+
+			if (wb_inode->size < stub->args.offset + req->write_size)
+				wb_inode->size = stub->args.offset
+				                 + req->write_size;
+		}
+		UNLOCK (&wb_inode->lock);
+
+		req->fd = fd_ref (stub->args.fd);
+
+		break;
+	case GF_FOP_READ:
+		req->ordering.off = stub->args.offset;
+		req->ordering.size = stub->args.size;
+
+		req->fd = fd_ref (stub->args.fd);
+
+		break;
+	case GF_FOP_TRUNCATE:
+		req->ordering.off = stub->args.offset;
+		req->ordering.size = 0; /* till infinity */
+		LOCK (&wb_inode->lock);
+		{
+			wb_inode->size = req->ordering.off;
+		}
+		UNLOCK (&wb_inode->lock);
+		break;
+	case GF_FOP_FTRUNCATE:
+		req->ordering.off = stub->args.offset;
+		req->ordering.size = 0; /* till infinity */
+		LOCK (&wb_inode->lock);
+		{
+			wb_inode->size = req->ordering.off;
+		}
+		UNLOCK (&wb_inode->lock);
+
+		req->fd = fd_ref (stub->args.fd);
+
+		break;
+	default:
+		break;
+	}
+
+        LOCK (&wb_inode->lock);
+        {
+                list_add_tail (&req->all, &wb_inode->all);
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_stat_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->stat, loc);
-        }
+		req->gen = wb_inode->gen;
 
-        return 0;
-unwind:
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
+                list_add_tail (&req->todo, &wb_inode->todo);
+		__wb_request_ref (req); /* for wind */
 
-        if (stub) {
-                call_stub_destroy (stub);
+		if (req->ordering.tempted) {
+			list_add_tail (&req->lie, &wb_inode->temptation);
+			__wb_request_ref (req); /* for unwind */
+		}
         }
+        UNLOCK (&wb_inode->lock);
 
-        if (iter_fd != NULL) {
-                fd_unref (iter_fd);
-        }
+out:
+	if (!req)
+		return _gf_false;
 
-        return 0;
+	return _gf_true;
 }
 
 
-int32_t
-wb_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *buf)
-{
-        wb_local_t   *local   = NULL;
-        wb_request_t *request = NULL;
-        wb_file_t    *file    = NULL;
-        int32_t       ret     = -1;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-
-        request = local->request;
-        if ((file != NULL) && (request != NULL)) {
-                wb_request_unref (request);
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        if (errno == ENOMEM) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
+gf_boolean_t
+wb_enqueue (wb_inode_t *wb_inode, call_stub_t *stub)
+{
+	return wb_enqueue_common (wb_inode, stub, 0);
+}
 
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
 
-        return 0;
+gf_boolean_t
+wb_enqueue_tempted (wb_inode_t *wb_inode, call_stub_t *stub)
+{
+	return wb_enqueue_common (wb_inode, stub, 1);
 }
 
 
-int32_t
-wb_fstat_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
+wb_inode_t *
+__wb_inode_create (xlator_t *this, inode_t *inode)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+        wb_inode_t *wb_inode = NULL;
+        wb_conf_t  *conf     = NULL;
 
-        STACK_WIND (frame, wb_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
-        return 0;
-}
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        conf = this->private;
 
+        wb_inode = GF_CALLOC (1, sizeof (*wb_inode), gf_wb_mt_wb_inode_t);
+        if (!wb_inode)
+                goto out;
 
-int32_t
-wb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        wb_file_t    *file     = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1;
-        int           op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
+        INIT_LIST_HEAD (&wb_inode->all);
+        INIT_LIST_HEAD (&wb_inode->todo);
+        INIT_LIST_HEAD (&wb_inode->liability);
+        INIT_LIST_HEAD (&wb_inode->temptation);
+        INIT_LIST_HEAD (&wb_inode->wip);
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+        wb_inode->this = this;
 
-        local->file = file;
+        wb_inode->window_conf = conf->window_size;
 
-        frame->local = local;
+        LOCK_INIT (&wb_inode->lock);
 
-        if (file) {
-                stub = fop_fstat_stub (frame, wb_fstat_helper, fd);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        __inode_ctx_put (inode, this, (uint64_t)(unsigned long)wb_inode);
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+out:
+        return wb_inode;
+}
 
-                /*
-                  FIXME:should the request queue be emptied in case of error?
-                */
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_fstat_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fstat, fd);
-        }
 
-        return 0;
+wb_inode_t *
+wb_inode_create (xlator_t *this, inode_t *inode)
+{
+        wb_inode_t *wb_inode = NULL;
 
-unwind:
-        STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
-        if (stub) {
-                call_stub_destroy (stub);
+        LOCK (&inode->lock);
+        {
+                wb_inode = __wb_inode_ctx_get (this, inode);
+                if (!wb_inode)
+                        wb_inode = __wb_inode_create (this, inode);
         }
+        UNLOCK (&inode->lock);
 
-        return 0;
+out:
+        return wb_inode;
 }
 
 
-int32_t
-wb_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
-{
-        wb_local_t   *local         = NULL;
-        wb_request_t *request       = NULL;
-        wb_file_t    *file          = NULL;
-        call_frame_t *process_frame = NULL;
-        int32_t       ret           = -1;
-        fd_t         *fd            = NULL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-        request = local->request;
-
-        if ((request != NULL) && (file != NULL)) {
-                process_frame = copy_frame (frame);
-                if (process_frame == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
-        }
+void
+wb_inode_destroy (wb_inode_t *wb_inode)
+{
+        GF_VALIDATE_OR_GOTO ("write-behind", wb_inode, out);
 
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
+        LOCK_DESTROY (&wb_inode->lock);
+        GF_FREE (wb_inode);
+out:
+        return;
+}
 
-        if (request) {
-                wb_request_unref (request);
-        }
 
-        if (process_frame != NULL) {
-                ret = wb_process_queue (process_frame, file);
-                if (ret == -1) {
-                        if ((errno == ENOMEM) && (file != NULL)) {
-                                LOCK (&file->lock);
-                                {
-                                        file->op_ret = -1;
-                                        file->op_errno = ENOMEM;
-                                }
-                                UNLOCK (&file->lock);
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
+void
+__wb_fulfill_request (wb_request_t *req)
+{
+	wb_inode_t *wb_inode = NULL;
 
-                STACK_DESTROY (process_frame->root);
-        }
+	wb_inode = req->wb_inode;
 
-        if (file) {
-                LOCK (&file->lock);
-                {
-                        fd = file->fd;
-                }
-                UNLOCK (&file->lock);
+	req->ordering.fulfilled = 1;
+	wb_inode->window_current -= req->total_size;
+	wb_inode->transit -= req->total_size;
 
-                fd_unref (fd);
-        }
+	if (!req->ordering.lied) {
+		/* TODO: fail the req->frame with error if
+		   necessary
+		*/
+	}
 
-        return 0;
+	__wb_request_unref (req);
 }
 
 
-static int32_t
-wb_truncate_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset)
+void
+wb_head_done (wb_request_t *head)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-
-        STACK_WIND (frame, wb_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
-
-        return 0;
+	wb_request_t *req = NULL;
+	wb_request_t *tmp     = NULL;
+	wb_inode_t   *wb_inode = NULL;
+
+	wb_inode = head->wb_inode;
+
+	LOCK (&wb_inode->lock);
+	{
+		list_for_each_entry_safe (req, tmp, &head->winds, winds) {
+			__wb_fulfill_request (req);
+		}
+		__wb_fulfill_request (head);
+	}
+	UNLOCK (&wb_inode->lock);
 }
 
 
-int32_t
-wb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
-{
-        wb_file_t    *file     = NULL;
-        fd_t         *iter_fd  = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        if (loc->inode) {
-                /*
-                  FIXME: fd_lookup extends life of fd till the execution of
-                  truncate_cbk
-                */
-                iter_fd = fd_lookup (loc->inode, frame->root->pid);
-                if (iter_fd) {
-                        if (!fd_ctx_get (iter_fd, this, &tmp_file)){
-                                file = (wb_file_t *)(long)tmp_file;
-                        } else {
-                                fd_unref (iter_fd);
-                        }
-                }
-        }
+void
+wb_fulfill_err (wb_request_t *head, int op_errno)
+{
+	wb_inode_t *wb_inode;
+	wb_request_t *req;
+
+	wb_inode = head->wb_inode;
+
+	/* for all future requests yet to arrive */
+	fd_ctx_set (head->fd, THIS, op_errno);
+
+	LOCK (&wb_inode->lock);
+	{
+		/* for all requests already arrived */
+		list_for_each_entry (req, &wb_inode->all, all) {
+			if (req->fd != head->fd)
+				continue;
+			req->op_ret = -1;
+			req->op_errno = op_errno;
+		}
+	}
+	UNLOCK (&wb_inode->lock);
+}
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
 
-        local->file = file;
+int
+wb_fulfill_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		struct iatt *postbuf, dict_t *xdata)
+{
+        wb_inode_t   *wb_inode   = NULL;
+        wb_request_t *head       = NULL;
 
-        frame->local = local;
-        if (file) {
-                stub = fop_truncate_stub (frame, wb_truncate_helper, loc,
-                                          offset);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	head = frame->local;
+	frame->local = NULL;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        wb_inode = head->wb_inode;
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_truncate_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc, offset);
-        }
+	if (op_ret == -1) {
+		wb_fulfill_err (head, op_errno);
+	} else if (op_ret < head->total_size) {
+		/*
+		 * We've encountered a short write, for whatever reason.
+		 * Set an EIO error for the next fop. This should be
+		 * valid for writev or flush (close).
+		 *
+		 * TODO: Retry the write so we can potentially capture
+		 * a real error condition (i.e., ENOSPC).
+		 */
+		wb_fulfill_err (head, EIO);
+	}
 
-        return 0;
+	wb_head_done (head);
 
-unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+        wb_process_queue (wb_inode);
 
-        if (stub) {
-                call_stub_destroy (stub);
-        }
+        STACK_DESTROY (frame->root);
 
         return 0;
 }
 
 
-int32_t
-wb_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-        wb_local_t   *local   = NULL;
-        wb_request_t *request = NULL;
-        wb_file_t    *file    = NULL;
-        int32_t       ret     = -1;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-        request = local->request;
-
-        if ((request != NULL) && (file != NULL)) {
-                wb_request_unref (request);
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        if (errno == ENOMEM) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
+#define WB_IOV_LOAD(vec, cnt, req, head) do {				\
+		memcpy (&vec[cnt], req->stub->args.vector,		\
+			(req->stub->args.count * sizeof(vec[0])));	\
+		cnt += req->stub->args.count;				\
+		head->total_size += req->write_size;			\
+	} while (0)
 
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf);
 
-        return 0;
-}
+int
+wb_fulfill_head (wb_inode_t *wb_inode, wb_request_t *head)
+{
+	struct iovec  vector[MAX_VECTOR_COUNT];
+	int           count    = 0;
+	wb_request_t *req      = NULL;
+	call_frame_t *frame    = NULL;
+        gf_boolean_t  fderr    = _gf_false;
+        xlator_t     *this     = NULL;
 
+        this = THIS;
 
-static int32_t
-wb_ftruncate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     off_t offset)
-{
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+        /* make sure head->total_size is updated before we run into any
+         * errors
+         */
 
-        STACK_WIND (frame, wb_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset);
-        return 0;
-}
+	WB_IOV_LOAD (vector, count, head, head);
 
+	list_for_each_entry (req, &head->winds, winds) {
+		WB_IOV_LOAD (vector, count, req, head);
 
-int32_t
-wb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
-{
-        wb_file_t    *file     = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1;
-        int           op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
+		if (iobref_merge (head->stub->args.iobref,
+				  req->stub->args.iobref))
+			goto err;
+	}
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
+        if (wb_fd_err (head->fd, this, NULL)) {
+                fderr = _gf_true;
+                goto err;
         }
 
-        local->file = file;
+	frame = create_frame (wb_inode->this, wb_inode->this->ctx->pool);
+	if (!frame)
+		goto err;
 
-        frame->local = local;
+	frame->root->lk_owner = head->lk_owner;
+	frame->local = head;
 
-        if (file) {
-                stub = fop_ftruncate_stub (frame, wb_ftruncate_helper, fd,
-                                           offset);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	LOCK (&wb_inode->lock);
+	{
+		wb_inode->transit += head->total_size;
+	}
+	UNLOCK (&wb_inode->lock);
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	STACK_WIND (frame, wb_fulfill_cbk, FIRST_CHILD (frame->this),
+		    FIRST_CHILD (frame->this)->fops->writev,
+		    head->fd, vector, count,
+		    head->stub->args.offset,
+		    head->stub->args.flags,
+		    head->stub->args.iobref, NULL);
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_ftruncate_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate, fd, offset);
+	return 0;
+err:
+        if (!fderr) {
+                /* frame creation failure */
+                fderr = ENOMEM;
+                wb_fulfill_err (head, fderr);
         }
 
-        return 0;
-
-unwind:
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL, NULL);
-
-        if (stub) {
-                call_stub_destroy (stub);
-        }
+	wb_head_done (head);
 
-        return 0;
+	return fderr;
 }
 
 
-int32_t
-wb_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                struct iatt *statpost)
-{
-        wb_local_t   *local         = NULL;
-        wb_request_t *request       = NULL;
-        call_frame_t *process_frame = NULL;
-        wb_file_t    *file          = NULL;
-        int32_t       ret           = -1;
-        fd_t         *fd            = NULL;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-        request = local->request;
-
-        if (request) {
-                process_frame = copy_frame (frame);
-                if (process_frame == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
-        }
+#define NEXT_HEAD(head, req) do {					\
+		if (head)						\
+			ret |= wb_fulfill_head (wb_inode, head);	\
+		head = req;						\
+		expected_offset = req->stub->args.offset +		\
+			req->write_size;				\
+		curr_aggregate = 0;					\
+		vector_count = 0;					\
+	} while (0)
 
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost);
 
-        if (request) {
-                wb_request_unref (request);
-        }
+int
+wb_fulfill (wb_inode_t *wb_inode, list_head_t *liabilities)
+{
+	wb_request_t  *req     = NULL;
+	wb_request_t  *head    = NULL;
+	wb_request_t  *tmp     = NULL;
+	wb_conf_t     *conf    = NULL;
+	off_t          expected_offset = 0;
+	size_t         curr_aggregate = 0;
+	size_t         vector_count = 0;
+        int            ret          = 0;
+
+	conf = wb_inode->this->private;
+
+	list_for_each_entry_safe (req, tmp, liabilities, winds) {
+		list_del_init (&req->winds);
+
+		if (!head) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if (req->fd != head->fd) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if (!is_same_lkowner (&req->lk_owner, &head->lk_owner)) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if (expected_offset != req->stub->args.offset) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if ((curr_aggregate + req->write_size) > conf->aggregate_size) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		if (vector_count + req->stub->args.count >
+		    MAX_VECTOR_COUNT) {
+			NEXT_HEAD (head, req);
+			continue;
+		}
+
+		list_add_tail (&req->winds, &head->winds);
+		curr_aggregate += req->write_size;
+		vector_count += req->stub->args.count;
+	}
+
+	if (head)
+		ret |= wb_fulfill_head (wb_inode, head);
+
+	return ret;
+}
 
-        if (request && (process_frame != NULL)) {
-                ret = wb_process_queue (process_frame, file);
-                if (ret == -1) {
-                        if ((errno == ENOMEM) && (file != NULL)) {
-                                LOCK (&file->lock);
-                                {
-                                        file->op_ret = -1;
-                                        file->op_errno = ENOMEM;
-                                }
-                                UNLOCK (&file->lock);
-                        }
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
 
-                STACK_DESTROY (process_frame->root);
-        }
+void
+wb_do_unwinds (wb_inode_t *wb_inode, list_head_t *lies)
+{
+	wb_request_t *req = NULL;
+	wb_request_t *tmp = NULL;
+	call_frame_t *frame = NULL;
+	struct iatt   buf = {0, };
 
-        if (file) {
-                LOCK (&file->lock);
-                {
-                        fd = file->fd;
-                }
-                UNLOCK (&file->lock);
+        list_for_each_entry_safe (req, tmp, lies, unwinds) {
+                frame = req->stub->frame;
 
-                fd_unref (fd);
+                STACK_UNWIND_STRICT (writev, frame, req->op_ret, req->op_errno,
+				     &buf, &buf, NULL); /* :O */
+		req->stub->frame = NULL;
+
+		list_del_init (&req->unwinds);
+                wb_request_unref (req);
         }
 
-        return 0;
+        return;
 }
 
 
-static int32_t
-wb_setattr_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid)
+void
+__wb_pick_unwinds (wb_inode_t *wb_inode, list_head_t *lies)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+        wb_request_t *req = NULL;
+        wb_request_t *tmp = NULL;
 
-        STACK_WIND (frame, wb_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid);
-        return 0;
-}
+	list_for_each_entry_safe (req, tmp, &wb_inode->temptation, lie) {
+		if (!req->ordering.fulfilled &&
+		    wb_inode->window_current > wb_inode->window_conf)
+			continue;
 
+		list_del_init (&req->lie);
+		list_move_tail (&req->unwinds, lies);
 
-int32_t
-wb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid)
-{
-        wb_file_t    *file     = NULL;
-        fd_t         *iter_fd  = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+		wb_inode->window_current += req->orig_size;
 
-        frame->local = local;
+		if (!req->ordering.fulfilled) {
+			/* burden increased */
+			list_add_tail (&req->lie, &wb_inode->liability);
 
-        if (!(valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME))) {
-                STACK_WIND (frame, wb_setattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr, loc, stbuf,
-                            valid);
-                goto out;
-        }
+			req->ordering.lied = 1;
 
-        if (loc->inode) {
-                /*
-                  FIXME: fd_lookup extends life of fd till the execution
-                  of wb_utimens_cbk
-                */
-                iter_fd = fd_lookup (loc->inode, frame->root->pid);
-                if (iter_fd) {
-                        if (!fd_ctx_get (iter_fd, this, &tmp_file)) {
-                                file = (wb_file_t *)(long)tmp_file;
-                        } else {
-                                fd_unref (iter_fd);
-                        }
-                }
+			wb_inode->gen++;
+		}
+	}
 
-        }
+        return;
+}
 
-        local->file = file;
 
-        if (file) {
-                stub = fop_setattr_stub (frame, wb_setattr_helper, loc, stbuf,
-                                         valid);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
+int
+__wb_collapse_small_writes (wb_request_t *holder, wb_request_t *req)
+{
+        char          *ptr    = NULL;
+        struct iobuf  *iobuf  = NULL;
+        struct iobref *iobref = NULL;
+        int            ret    = -1;
+        ssize_t        required_size = 0;
+        size_t         holder_len = 0;
+        size_t         req_len = 0;
+
+        if (!holder->iobref) {
+                holder_len = iov_length (holder->stub->args.vector,
+                                         holder->stub->args.count);
+                req_len = iov_length (req->stub->args.vector,
+                                      req->stub->args.count);
+
+                required_size = max ((THIS->ctx->page_size),
+                                     (holder_len + req_len));
+                iobuf = iobuf_get2 (req->wb_inode->this->ctx->iobuf_pool,
+                                    required_size);
+                if (iobuf == NULL) {
+                        goto out;
                 }
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
+                iobref = iobref_new ();
+                if (iobref == NULL) {
+                        iobuf_unref (iobuf);
+                        goto out;
                 }
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
+                ret = iobref_add (iobref, iobuf);
+                if (ret != 0) {
+                        gf_log (req->wb_inode->this->name, GF_LOG_WARNING,
+                                "cannot add iobuf (%p) into iobref (%p)",
+                                iobuf, iobref);
+                        iobuf_unref (iobuf);
+                        iobref_unref (iobref);
+                        goto out;
                 }
-        } else {
-                STACK_WIND (frame, wb_setattr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setattr, loc, stbuf,
-                            valid);
-        }
 
-        return 0;
-unwind:
-        STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
+                iov_unload (iobuf->ptr, holder->stub->args.vector,
+                            holder->stub->args.count);
+                holder->stub->args.vector[0].iov_base = iobuf->ptr;
+		holder->stub->args.count = 1;
 
-        if (stub) {
-                call_stub_destroy (stub);
+                iobref_unref (holder->stub->args.iobref);
+                holder->stub->args.iobref = iobref;
+
+                iobuf_unref (iobuf);
+
+                holder->iobref = iobref_ref (iobref);
         }
+
+        ptr = holder->stub->args.vector[0].iov_base + holder->write_size;
+
+        iov_unload (ptr, req->stub->args.vector,
+                    req->stub->args.count);
+
+        holder->stub->args.vector[0].iov_len += req->write_size;
+        holder->write_size += req->write_size;
+        holder->ordering.size += req->write_size;
+
+        ret = 0;
 out:
-        return 0;
+        return ret;
 }
 
+
 void
-wb_disable_all (xlator_t *this, fd_t *origfd)
+__wb_preprocess_winds (wb_inode_t *wb_inode)
 {
-        inode_t *inode    = NULL;
-        fd_t    *otherfd  = NULL;
-        uint64_t tmp_file = 0;
-
-        inode = origfd->inode;
-        LOCK(&inode->lock);
-        list_for_each_entry (otherfd, &inode->fd_list, inode_list) {
-                if (otherfd == origfd) {
-                        continue;
-                }
-                if (fd_ctx_get(otherfd,this,&tmp_file)) {
+        off_t         offset_expected = 0;
+        ssize_t       space_left      = 0;
+	wb_request_t *req             = NULL;
+	wb_request_t *tmp             = NULL;
+	wb_request_t *holder          = NULL;
+	wb_conf_t    *conf            = NULL;
+        int           ret             = 0;
+	ssize_t       page_size       = 0;
+
+	/* With asynchronous IO from a VM guest (as a file), there
+	   can be two sequential writes happening in two regions
+	   of the file. But individual (broken down) IO requests
+	   can arrive interleaved.
+
+	   TODO: cycle for each such sequence sifting
+	   through the interleaved ops
+	*/
+
+	page_size = wb_inode->this->ctx->page_size;
+	conf = wb_inode->this->private;
+
+        list_for_each_entry_safe (req, tmp, &wb_inode->todo, todo) {
+		if (!req->ordering.tempted) {
+			if (holder) {
+				if (wb_requests_conflict (holder, req))
+					/* do not hold on write if a
+					   dependent write is in queue */
+					holder->ordering.go = 1;
+			}
+			/* collapse only non-sync writes */
+			continue;
+		} else if (!holder) {
+			/* holder is always a non-sync write */
+			holder = req;
+			continue;
+		}
+
+		offset_expected = holder->stub->args.offset
+			+ holder->write_size;
+
+		if (req->stub->args.offset != offset_expected) {
+			holder->ordering.go = 1;
+			holder = req;
+			continue;
+		}
+
+		if (!is_same_lkowner (&req->lk_owner, &holder->lk_owner)) {
+			holder->ordering.go = 1;
+			holder = req;
+			continue;
+		}
+
+                if (req->fd != holder->fd) {
+                        holder->ordering.go = 1;
+                        holder = req;
                         continue;
                 }
-                gf_log(this->name,GF_LOG_DEBUG,
-                       "disabling wb on %p because %p is O_SYNC",
-                       otherfd, origfd);
-                ((wb_file_t *)(long)tmp_file)->disabled = 1;
-        }
-        UNLOCK(&inode->lock);
-}
 
-int32_t
-wb_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd)
-{
-        int32_t     wbflags = 0, flags = 0;
-        wb_file_t  *file    = NULL;
-        wb_conf_t  *conf    = NULL;
-        wb_local_t *local   = NULL;
+		space_left = page_size - holder->write_size;
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, out, op_errno,
-                                        EINVAL);
+		if (space_left < req->write_size) {
+			holder->ordering.go = 1;
+			holder = req;
+			continue;
+		}
 
-        conf = this->private;
+		ret = __wb_collapse_small_writes (holder, req);
+		if (ret)
+			continue;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, local, out, op_errno,
-                                        EINVAL);
+		/* collapsed request is as good as wound
+		   (from its p.o.v)
+		*/
+		list_del_init (&req->todo);
+		__wb_fulfill_request (req);
 
-        flags = local->flags;
-        wbflags = local->wbflags;
+               /* Only the last @holder in queue which
 
-        if (op_ret != -1) {
-                file = wb_file_create (this, fd, flags);
-                if (file == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
+                  - does not have any non-buffered-writes following it
+                  - has not yet filled its capacity
 
-                LOCK (&file->lock);
-                {
-                        /* If O_DIRECT then, we disable caching */
-                        if (((flags & O_DIRECT) == O_DIRECT)
-                            || ((flags & O_ACCMODE) == O_RDONLY)
-                            || (((flags & O_SYNC) == O_SYNC)
-                                && conf->enable_O_SYNC == _gf_true)) {
-                                gf_log(this->name,GF_LOG_DEBUG,
-                                       "disabling wb on %p", fd);
-                                file->disabled = 1;
-                                wb_disable_all(this,fd);
-                        }
-
-                        else if (wbflags & GF_OPEN_NOWB) {
-                                file->disabled = 1;
-                        }
-                }
-                UNLOCK (&file->lock);
+                  does not get its 'go' set, in anticipation of the arrival
+                  of consecutive smaller writes.
+               */
         }
 
-out:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
-        return 0;
+	/* but if trickling writes are enabled, then do not hold back
+	   writes if there are no outstanding requests
+	*/
+
+	if (conf->trickling_writes && !wb_inode->transit && holder)
+		holder->ordering.go = 1;
+
+        return;
 }
 
 
-int32_t
-wb_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+void
+__wb_pick_winds (wb_inode_t *wb_inode, list_head_t *tasks,
+		 list_head_t *liabilities)
 {
-        wb_local_t *local    = NULL;
-        int32_t     op_errno = EINVAL;
+	wb_request_t *req = NULL;
+	wb_request_t *tmp = NULL;
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+	list_for_each_entry_safe (req, tmp, &wb_inode->todo, todo) {
+		if (wb_liability_has_conflict (wb_inode, req))
+			continue;
 
-        local->flags = flags;
-        local->wbflags = wbflags;
+		if (req->ordering.tempted && !req->ordering.go)
+			/* wait some more */
+			continue;
 
-        frame->local = local;
+		if (req->stub->fop == GF_FOP_WRITE) {
+			if (wb_wip_has_conflict (wb_inode, req))
+				continue;
 
-        STACK_WIND (frame, wb_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
-        return 0;
+			list_add_tail (&req->wip, &wb_inode->wip);
 
-unwind:
-        STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
-        return 0;
-}
+			if (!req->ordering.tempted)
+				/* unrefed in wb_writev_cbk */
+				req->stub->frame->local =
+					__wb_request_ref (req);
+		}
 
+		list_del_init (&req->todo);
 
-int32_t
-wb_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent)
-{
-        long       flags = 0;
-        wb_file_t *file  = NULL;
-        wb_conf_t *conf  = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, out,
-                                        op_errno, EINVAL);
+		if (req->ordering.tempted)
+			list_add_tail (&req->winds, liabilities);
+		else
+			list_add_tail (&req->winds, tasks);
+	}
+}
 
-        conf = this->private;
-        if (op_ret != -1) {
-                if (frame->local) {
-                        flags = (long) frame->local;
-                }
 
-                file = wb_file_create (this, fd, flags);
-                if (file == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
+void
+wb_do_winds (wb_inode_t *wb_inode, list_head_t *tasks)
+{
+	wb_request_t *req = NULL;
+	wb_request_t *tmp = NULL;
 
-                LOCK (&file->lock);
-                {
-                        /* If O_DIRECT then, we disable caching */
-                        if (frame->local) {
-                                if (((flags & O_DIRECT) == O_DIRECT)
-                                    || ((flags & O_ACCMODE) == O_RDONLY)
-                                    || (((flags & O_SYNC) == O_SYNC)
-                                        && (conf->enable_O_SYNC == _gf_true))) {
-                                        file->window_conf = 0;
-                                }
-                        }
-                }
-                UNLOCK (&file->lock);
-        }
+	list_for_each_entry_safe (req, tmp, tasks, winds) {
+		list_del_init (&req->winds);
 
-        frame->local = NULL;
+		call_resume (req->stub);
 
-out:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
-        return 0;
+		wb_request_unref (req);
+	}
 }
 
 
-int32_t
-wb_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params)
+void
+wb_process_queue (wb_inode_t *wb_inode)
 {
-        int32_t op_errno = EINVAL;
+        list_head_t tasks  = {0, };
+	list_head_t lies = {0, };
+	list_head_t liabilities = {0, };
+        int         retry       = 0;
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
+        INIT_LIST_HEAD (&tasks);
+        INIT_LIST_HEAD (&lies);
+        INIT_LIST_HEAD (&liabilities);
 
-        frame->local = (void *)(long)flags;
+        do {
+                LOCK (&wb_inode->lock);
+                {
+                        __wb_preprocess_winds (wb_inode);
 
-        STACK_WIND (frame, wb_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
-        return 0;
+                        __wb_pick_winds (wb_inode, &tasks, &liabilities);
 
-unwind:
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
-        return 0;
-}
+                        __wb_pick_unwinds (wb_inode, &lies);
 
+                }
+                UNLOCK (&wb_inode->lock);
 
-/* Mark all the contiguous write requests for winding starting from head of
- * request list. Stops marking at the first non-write request found. If
- * file is opened with O_APPEND, make sure all the writes marked for winding
- * will fit into a single write call to server.
- */
-size_t
-__wb_mark_wind_all (wb_file_t *file, list_head_t *list, list_head_t *winds)
-{
-        wb_request_t *request         = NULL;
-        size_t        size            = 0;
-        char          first_request   = 1;
-        off_t         offset_expected = 0;
-        wb_conf_t    *conf            = NULL;
-        int           count           = 0;
+                wb_do_unwinds (wb_inode, &lies);
 
-        GF_VALIDATE_OR_GOTO ("write-behind", file, out);
-        GF_VALIDATE_OR_GOTO (file->this->name, list, out);
-        GF_VALIDATE_OR_GOTO (file->this->name, winds, out);
+                wb_do_winds (wb_inode, &tasks);
 
-        conf = file->this->private;
+                /* fd might've been marked bad due to previous errors.
+                 * Since, caller of wb_process_queue might be the last fop on
+                 * inode, make sure we keep processing request queue, till there
+                 * are no requests left.
+                 */
+                retry = wb_fulfill (wb_inode, &liabilities);
+        } while (retry);
 
-        list_for_each_entry (request, list, list)
-        {
-                if ((request->stub == NULL)
-                    || (request->stub->fop != GF_FOP_WRITE)) {
-                        break;
-                }
+        return;
+}
 
-                if (!request->flags.write_request.stack_wound) {
-                        if (first_request) {
-                                first_request = 0;
-                                offset_expected
-                                        = request->stub->args.writev.off;
-                        }
-
-                        if (request->stub->args.writev.off != offset_expected) {
-                                break;
-                        }
-
-                        if ((file->flags & O_APPEND)
-                            && (((size + request->write_size)
-                                 > conf->aggregate_size)
-                                || ((count + request->stub->args.writev.count)
-                                    > MAX_VECTOR_COUNT))) {
-                                break;
-                        }
-
-                        size += request->write_size;
-                        offset_expected += request->write_size;
-                        file->aggregate_current -= request->write_size;
-                        count += request->stub->args.writev.count;
-
-                        request->flags.write_request.stack_wound = 1;
-                        list_add_tail (&request->winds, winds);
-                }
-        }
 
-out:
-        return size;
+void
+wb_set_inode_size(wb_inode_t *wb_inode, struct iatt *postbuf)
+{
+	GF_ASSERT (wb_inode);
+	GF_ASSERT (postbuf);
+
+	LOCK (&wb_inode->lock);
+	{
+		wb_inode->size = postbuf->ia_size;
+	}
+	UNLOCK (&wb_inode->lock);
 }
 
 
-int32_t
-__wb_can_wind (list_head_t *list, char *other_fop_in_queue,
-               char *non_contiguous_writes, char *incomplete_writes,
-               char *wind_all)
+int
+wb_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	       int32_t op_ret, int32_t op_errno,
+	       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        wb_request_t *request         = NULL;
-        char          first_request   = 1;
-        off_t         offset_expected = 0;
-        int32_t       ret             = -1;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", list, out);
+	wb_request_t *req = NULL;
+	wb_inode_t *wb_inode;
 
-        list_for_each_entry (request, list, list)
-        {
-                if ((request->stub == NULL)
-                    || (request->stub->fop != GF_FOP_WRITE)) {
-                        if (request->stub && other_fop_in_queue) {
-                                *other_fop_in_queue = 1;
-                        }
-                        break;
-                }
+	req = frame->local;
+	frame->local = NULL;
+	wb_inode = req->wb_inode;
 
-                if (request->flags.write_request.stack_wound
-                    && !request->flags.write_request.got_reply
-                    && (incomplete_writes != NULL)) {
-                        *incomplete_writes = 1;
-                        break;
-                }
+	wb_request_unref (req);
 
-                if (!request->flags.write_request.stack_wound) {
-                        if (first_request) {
-                                char flush = 0;
-                                first_request = 0;
-                                offset_expected
-                                        = request->stub->args.writev.off;
-
-                                flush = request->flags.write_request.flush_all;
-                                if (wind_all != NULL) {
-                                        *wind_all = flush;
-                                }
-                        }
-
-                        if (offset_expected != request->stub->args.writev.off) {
-                                if (non_contiguous_writes) {
-                                        *non_contiguous_writes = 1;
-                                }
-                                break;
-                        }
-
-                        offset_expected += request->write_size;
-                }
-        }
+	/* requests could be pending while this was in progress */
+	wb_process_queue(wb_inode);
 
-        ret = 0;
-out:
-        return ret;
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+			     xdata);
+	return 0;
 }
 
 
-ssize_t
-__wb_mark_winds (list_head_t *list, list_head_t *winds, size_t aggregate_conf,
-                 char enable_trickling_writes)
+int
+wb_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  struct iovec *vector, int32_t count, off_t offset,
+		  uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        size_t        size                   = 0;
-        char          other_fop_in_queue     = 0;
-        char          incomplete_writes      = 0;
-        char          non_contiguous_writes  = 0;
-        wb_request_t *request                = NULL;
-        wb_file_t    *file                   = NULL;
-        char          wind_all               = 0;
-        int32_t       ret                    = 0;
+	STACK_WIND (frame, wb_writev_cbk,
+		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
+		    fd, vector, count, offset, flags, iobref, xdata);
+	return 0;
+}
 
-        GF_VALIDATE_OR_GOTO ("write-behind", list, out);
-        GF_VALIDATE_OR_GOTO ("write-behind", winds, out);
 
-        if (list_empty (list)) {
-                goto out;
-        }
+int
+wb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
+{
+        wb_inode_t   *wb_inode      = NULL;
+	wb_conf_t    *conf          = NULL;
+        gf_boolean_t  wb_disabled   = 0;
+        call_stub_t  *stub          = NULL;
+        int           ret           = -1;
+        int32_t       op_errno      = EINVAL;
+	int           o_direct      = O_DIRECT;
 
-        request = list_entry (list->next, typeof (*request), list);
-        file = request->file;
+	conf = this->private;
 
-        ret = __wb_can_wind (list, &other_fop_in_queue,
-                             &non_contiguous_writes, &incomplete_writes,
-                             &wind_all);
-        if (ret == -1) {
-                gf_log (file->this->name, GF_LOG_WARNING,
-                        "cannot decide whether to wind or not");
-                goto out;
+        if (wb_fd_err (fd, this, &op_errno)) {
+                goto unwind;
         }
 
-        if (!incomplete_writes && ((enable_trickling_writes)
-                                   || (wind_all) || (non_contiguous_writes)
-                                   || (other_fop_in_queue)
-                                   || (file->aggregate_current
-                                       >= aggregate_conf))) {
-                size = __wb_mark_wind_all (file, list, winds);
-        }
+        wb_inode = wb_inode_create (this, fd->inode);
+	if (!wb_inode) {
+		op_errno = ENOMEM;
+		goto unwind;
+	}
 
-out:
-        return size;
-}
+	if (!conf->strict_O_DIRECT)
+		o_direct = 0;
 
+	if (fd->flags & (O_SYNC|O_DSYNC|o_direct))
+		wb_disabled = 1;
 
-size_t
-__wb_mark_unwind_till (list_head_t *list, list_head_t *unwinds, size_t size)
-{
-        size_t        written_behind = 0;
-        wb_request_t *request        = NULL;
-        wb_file_t    *file           = NULL;
+	if (flags & (O_SYNC|O_DSYNC|o_direct))
+		wb_disabled = 1;
 
-        if (list_empty (list)) {
-                goto out;
+	if (wb_disabled)
+		stub = fop_writev_stub (frame, wb_writev_helper, fd, vector,
+					count, offset, flags, iobref, xdata);
+	else
+		stub = fop_writev_stub (frame, NULL, fd, vector, count, offset,
+					flags, iobref, xdata);
+        if (!stub) {
+                op_errno = ENOMEM;
+                goto unwind;
         }
 
-        request = list_entry (list->next, typeof (*request), list);
-        file = request->file;
+	if (wb_disabled)
+		ret = wb_enqueue (wb_inode, stub);
+	else
+		ret = wb_enqueue_tempted (wb_inode, stub);
 
-        list_for_each_entry (request, list, list)
-        {
-                if ((request->stub == NULL)
-                    || (request->stub->fop != GF_FOP_WRITE)) {
-                        continue;
-                }
+	if (!ret) {
+		op_errno = ENOMEM;
+		goto unwind;
+	}
 
-                if (written_behind <= size) {
-                        if (!request->flags.write_request.write_behind) {
-                                written_behind += request->write_size;
-                                request->flags.write_request.write_behind = 1;
-                                list_add_tail (&request->unwinds, unwinds);
-
-                                if (!request->flags.write_request.got_reply) {
-                                        file->window_current
-                                                += request->write_size;
-                                }
-                        }
-                } else {
-                        break;
-                }
-        }
+        wb_process_queue (wb_inode);
 
-out:
-        return written_behind;
-}
-
-
-void
-__wb_mark_unwinds (list_head_t *list, list_head_t *unwinds)
-{
-        wb_request_t *request = NULL;
-        wb_file_t    *file    = NULL;
+        return 0;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", list, out);
-        GF_VALIDATE_OR_GOTO ("write-behind", unwinds, out);
+unwind:
+        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        if (list_empty (list)) {
-                goto out;
-        }
+        if (stub)
+                call_stub_destroy (stub);
 
-        request = list_entry (list->next, typeof (*request), list);
-        file = request->file;
+        return 0;
+}
 
-        if (file->window_current <= file->window_conf) {
-                __wb_mark_unwind_till (list, unwinds,
-                                       file->window_conf
-                                       - file->window_current);
-        }
 
-out:
-        return;
+int
+wb_readv_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, uint32_t flags, dict_t *xdata)
+{
+        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                    xdata);
+        return 0;
 }
 
 
-uint32_t
-__wb_get_other_requests (list_head_t *list, list_head_t *other_requests)
+int
+wb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        wb_request_t *request = NULL;
-        uint32_t      count   = 0;
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", list, out);
-        GF_VALIDATE_OR_GOTO ("write-behind", other_requests, out);
+        wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        list_for_each_entry (request, list, list) {
-                if ((request->stub == NULL)
-                    || (request->stub->fop == GF_FOP_WRITE)) {
-                        break;
-                }
+	stub = fop_readv_stub (frame, wb_readv_helper, fd, size,
+			       offset, flags, xdata);
+	if (!stub)
+		goto unwind;
 
-                if (!request->flags.other_requests.marked_for_resume) {
-                        request->flags.other_requests.marked_for_resume = 1;
-                        list_add_tail (&request->other_requests,
-                                       other_requests);
-                        count++;
-                }
-        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-out:
-        return count;
+	wb_process_queue (wb_inode);
+
+        return 0;
+
+unwind:
+        STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
+                             NULL);
+        return 0;
+
+noqueue:
+        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                    xdata);
+        return 0;
 }
 
 
-int32_t
-wb_stack_unwind (list_head_t *unwinds)
+int
+wb_flush_bg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        struct iatt   buf     = {0,};
-        wb_request_t *request = NULL, *dummy = NULL;
-        call_frame_t *frame   = NULL;
-        wb_local_t   *local   = NULL;
-        int           ret     = 0, write_requests_removed = 0;
+        STACK_DESTROY (frame->root);
+        return 0;
+}
 
-        GF_VALIDATE_OR_GOTO ("write-behind", unwinds, out);
 
-        list_for_each_entry_safe (request, dummy, unwinds, unwinds) {
-                frame = request->stub->frame;
-                local = frame->local;
+int
+wb_flush_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        wb_conf_t    *conf        = NULL;
+        wb_inode_t   *wb_inode    = NULL;
+        call_frame_t *bg_frame    = NULL;
+	int32_t       op_errno    = 0;
+	int           op_ret      = 0;
 
-                STACK_UNWIND (frame, local->op_ret, local->op_errno, &buf,
-                              &buf);
+        conf = this->private;
 
-                ret = wb_request_unref (request);
-                if (ret == 0) {
-                        write_requests_removed++;
-                }
-        }
+	wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode) {
+		op_ret = -1;
+		op_errno = EINVAL;
+		goto unwind;
+	}
+
+	if (wb_fd_err (fd, this, &op_errno)) {
+		op_ret = -1;
+		goto unwind;
+	}
+
+        if (conf->flush_behind)
+		goto flushbehind;
+
+	STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->flush, fd, xdata);
+	return 0;
+
+flushbehind:
+	bg_frame = copy_frame (frame);
+	if (!bg_frame) {
+		op_ret = -1;
+		op_errno = ENOMEM;
+		goto unwind;
+	}
+
+	STACK_WIND (bg_frame, wb_flush_bg_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->flush, fd, xdata);
+	/* fall through */
+unwind:
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, NULL);
 
-out:
-        return write_requests_removed;
+        return 0;
 }
 
 
-int32_t
-wb_resume_other_requests (call_frame_t *frame, wb_file_t *file,
-                          list_head_t *other_requests)
+int
+wb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t       ret          = -1;
-        wb_request_t *request      = NULL, *dummy = NULL;
-        int32_t       fops_removed = 0;
-        char          wind         = 0;
+        wb_inode_t   *wb_inode     = NULL;
         call_stub_t  *stub         = NULL;
 
-        GF_VALIDATE_OR_GOTO ((file ? file->this->name : "write-behind"), frame,
-                             out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, file, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, other_requests, out);
+        wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        if (list_empty (other_requests)) {
-                ret = 0;
-                goto out;
-        }
+	stub = fop_flush_stub (frame, wb_flush_helper, fd, xdata);
+	if (!stub)
+		goto unwind;
 
-        list_for_each_entry_safe (request, dummy, other_requests,
-                                  other_requests) {
-                wind = request->stub->wind;
-                stub = request->stub;
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-                LOCK (&file->lock);
-                {
-                        request->stub = NULL;
-                }
-                UNLOCK (&file->lock);
+        wb_process_queue (wb_inode);
 
-                if (!wind) {
-                        wb_request_unref (request);
-                        fops_removed++;
-                }
+        return 0;
 
-                call_resume (stub);
-        }
+unwind:
+        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
 
-        ret = 0;
+        return 0;
 
-        if (fops_removed > 0) {
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
+noqueue:
+        STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->flush, fd, xdata);
+        return 0;
+}
 
-out:
-        return ret;
+
+
+int
+wb_fsync_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 int32_t datasync, dict_t *xdata)
+{
+        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+        return 0;
 }
 
 
-int32_t
-wb_do_ops (call_frame_t *frame, wb_file_t *file, list_head_t *winds,
-           list_head_t *unwinds, list_head_t *other_requests)
+int
+wb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
 {
-        int32_t ret = -1, write_requests_removed = 0;
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
+	int32_t       op_errno     = EINVAL;
 
-        GF_VALIDATE_OR_GOTO ((file ? file->this->name : "write-behind"),
-                             frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, file, out);
+	if (wb_fd_err (fd, this, &op_errno))
+		goto unwind;
 
-        ret = wb_stack_unwind (unwinds);
+        wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        write_requests_removed = ret;
+	stub = fop_fsync_stub (frame, wb_fsync_helper, fd, datasync, xdata);
+	if (!stub)
+		goto unwind;
 
-        ret = wb_sync (frame, file, winds);
-        if (ret == -1) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "syncing of write requests failed");
-        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-        ret = wb_resume_other_requests (frame, file, other_requests);
-        if (ret == -1) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "cannot resume non-write requests in request queue");
-        }
+	wb_process_queue (wb_inode);
 
-        /* wb_stack_unwind does wb_request_unref after unwinding a write
-         * request. Hence if a write-request was just freed in wb_stack_unwind,
-         * we have to process request queue once again to unblock requests
-         * blocked on the writes just unwound.
-         */
-        if (write_requests_removed > 0) {
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
+        return 0;
 
-out:
-        return ret;
+unwind:
+        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+
+noqueue:
+        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+        return 0;
 }
 
 
-inline int
-__wb_copy_into_holder (wb_request_t *holder, wb_request_t *request)
+int
+wb_stat_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        char          *ptr    = NULL;
-        struct iobuf  *iobuf  = NULL;
-        struct iobref *iobref = NULL;
-        int            ret    = -1;
+        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+}
 
-        if (holder->flags.write_request.virgin) {
-                /* TODO: check the required size */
-                iobuf = iobuf_get (request->file->this->ctx->iobuf_pool);
-                if (iobuf == NULL) {
-                        goto out;
-                }
 
-                iobref = iobref_new ();
-                if (iobref == NULL) {
-                        iobuf_unref (iobuf);
-                        goto out;
-                }
-
-                ret = iobref_add (iobref, iobuf);
-                if (ret != 0) {
-                        iobuf_unref (iobuf);
-                        iobref_unref (iobref);
-                        gf_log (request->file->this->name, GF_LOG_WARNING,
-                                "cannot add iobuf (%p) into iobref (%p)",
-                                iobuf, iobref);
-                        goto out;
-                }
+int
+wb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-                iov_unload (iobuf->ptr, holder->stub->args.writev.vector,
-                            holder->stub->args.writev.count);
-                holder->stub->args.writev.vector[0].iov_base = iobuf->ptr;
 
-                iobref_unref (holder->stub->args.writev.iobref);
-                holder->stub->args.writev.iobref = iobref;
+	wb_inode = wb_inode_ctx_get (this, loc->inode);
+        if (!wb_inode)
+		goto noqueue;
 
-                iobuf_unref (iobuf);
+	stub = fop_stat_stub (frame, wb_stat_helper, loc, xdata);
+	if (!stub)
+		goto unwind;
 
-                holder->flags.write_request.virgin = 0;
-        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-        ptr = holder->stub->args.writev.vector[0].iov_base + holder->write_size;
+	wb_process_queue (wb_inode);
 
-        iov_unload (ptr, request->stub->args.writev.vector,
-                    request->stub->args.writev.count);
+        return 0;
 
-        holder->stub->args.writev.vector[0].iov_len += request->write_size;
-        holder->write_size += request->write_size;
+unwind:
+        STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL, NULL);
 
-        request->flags.write_request.stack_wound = 1;
-        list_move_tail (&request->list, &request->file->passive_requests);
+        if (stub)
+                call_stub_destroy (stub);
+        return 0;
 
-        ret = 0;
-out:
-        return ret;
+noqueue:
+        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+	return 0;
 }
 
 
-/* this procedure assumes that write requests have only one vector to write */
-void
-__wb_collapse_write_bufs (list_head_t *requests, size_t page_size)
+int
+wb_fstat_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        off_t         offset_expected = 0;
-        size_t        space_left      = 0;
-        wb_request_t *request         = NULL, *tmp = NULL, *holder = NULL;
-        int           ret             = 0;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", requests, out);
-
-        list_for_each_entry_safe (request, tmp, requests, list) {
-                if ((request->stub == NULL)
-                    || (request->stub->fop != GF_FOP_WRITE)
-                    || (request->flags.write_request.stack_wound)) {
-                        holder = NULL;
-                        continue;
-                }
-
-                if (request->flags.write_request.write_behind) {
-                        if (holder == NULL) {
-                                holder = request;
-                                continue;
-                        }
-
-                        offset_expected = holder->stub->args.writev.off
-                                + holder->write_size;
-
-                        if (request->stub->args.writev.off != offset_expected) {
-                                holder = request;
-                                continue;
-                        }
-
-                        space_left = page_size - holder->write_size;
-
-                        if (space_left >= request->write_size) {
-                                ret = __wb_copy_into_holder (holder, request);
-                                if (ret != 0) {
-                                        break;
-                                }
-
-                                __wb_request_unref (request);
-                        } else {
-                                holder = request;
-                        }
-                } else {
-                        break;
-                }
-        }
-
-out:
-        return;
+        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
 }
 
 
-int32_t
-wb_process_queue (call_frame_t *frame, wb_file_t *file)
+int
+wb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        list_head_t winds  = {0, }, unwinds = {0, }, other_requests = {0, };
-        size_t      size   = 0;
-        wb_conf_t  *conf   = NULL;
-        uint32_t    count  = 0;
-        int32_t     ret    = -1;
+        wb_inode_t   *wb_inode     = NULL;
+	call_stub_t  *stub         = NULL;
 
-        INIT_LIST_HEAD (&winds);
-        INIT_LIST_HEAD (&unwinds);
-        INIT_LIST_HEAD (&other_requests);
 
-        GF_VALIDATE_OR_GOTO ((file ? file->this->name : "write-behind"), frame,
-                             out);
-        GF_VALIDATE_OR_GOTO (file->this->name, frame, out);
+        wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        conf = file->this->private;
-        GF_VALIDATE_OR_GOTO (file->this->name, conf, out);
+	stub = fop_fstat_stub (frame, wb_fstat_helper, fd, xdata);
+	if (!stub)
+		goto unwind;
 
-        size = conf->aggregate_size;
-        LOCK (&file->lock);
-        {
-                /*
-                 * make sure requests are marked for unwinding and adjacent
-                 * contiguous write buffers (each of size less than that of
-                 * an iobuf) are packed properly so that iobufs are filled to
-                 * their maximum capacity, before calling __wb_mark_winds.
-                 */
-                __wb_mark_unwinds (&file->request, &unwinds);
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-                __wb_collapse_write_bufs (&file->request,
-                                          file->this->ctx->page_size);
+	wb_process_queue (wb_inode);
 
-                count = __wb_get_other_requests (&file->request,
-                                                 &other_requests);
-
-                if (count == 0) {
-                        __wb_mark_winds (&file->request, &winds, size,
-                                         conf->enable_trickling_writes);
-                }
+        return 0;
 
-        }
-        UNLOCK (&file->lock);
+unwind:
+        STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL, NULL);
 
-        ret = wb_do_ops (frame, file, &winds, &unwinds, &other_requests);
+        if (stub)
+                call_stub_destroy (stub);
+        return 0;
 
-out:
-        return ret;
+noqueue:
+        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
 }
 
 
 int32_t
-wb_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+wb_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        GF_ASSERT (frame);
+        GF_ASSERT (frame->local);
+
+        if (op_ret == 0)
+                wb_set_inode_size (frame->local, postbuf);
+
+        frame->local = NULL;
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
         return 0;
 }
 
 
-int32_t
-wb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref)
+int
+wb_truncate_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    off_t offset, dict_t *xdata)
 {
-        wb_file_t    *file          = NULL;
-        char          wb_disabled   = 0;
-        call_frame_t *process_frame = NULL;
-        size_t        size          = 0;
-        uint64_t      tmp_file      = 0;
-        call_stub_t  *stub          = NULL;
-        wb_local_t   *local         = NULL;
-        wb_request_t *request       = NULL;
-        int32_t       ret           = -1;
-        int32_t       op_ret        = -1, op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("write-behind", this, unwind, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, fd, unwind, op_errno,
-                                        EINVAL);
-
-        if (vector != NULL)
-                size = iov_length (vector, count);
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
-
-        if (file != NULL) {
-                LOCK (&file->lock);
-                {
-                        op_ret = file->op_ret;
-                        op_errno = file->op_errno;
-
-                        file->op_ret = 0;
-
-                        if ((op_ret == 0)
-                            && (file->disabled || file->disable_till)) {
-                                if (size > file->disable_till) {
-                                        file->disable_till = 0;
-                                } else {
-                                        file->disable_till -= size;
-                                }
-                                wb_disabled = 1;
-                        }
-                }
-                UNLOCK (&file->lock);
-        } else {
-                wb_disabled = 1;
-        }
-
-        if (op_ret == -1) {
-                STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, NULL,
-                                     NULL);
-                return 0;
-        }
-
-        if (wb_disabled) {
-                STACK_WIND (frame, wb_writev_cbk, FIRST_CHILD (frame->this),
-                            FIRST_CHILD (frame->this)->fops->writev,
-                            fd, vector, count, offset, flags, iobref);
-                return 0;
-        }
+        STACK_WIND (frame, wb_truncate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        return 0;
+}
 
-        process_frame = copy_frame (frame);
-        if (process_frame == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+int
+wb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
+	call_stub_t  *stub         = NULL;
 
-        frame->local = local;
-        local->file = file;
+	wb_inode = wb_inode_create (this, loc->inode);
+	if (!wb_inode)
+		goto unwind;
 
-        stub = fop_writev_stub (frame, NULL, fd, vector, count, offset, flags,
-                                iobref);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+	frame->local = wb_inode;
 
-        request = wb_enqueue (file, stub);
-        if (request == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+	stub = fop_truncate_stub (frame, wb_truncate_helper, loc,
+				  offset, xdata);
+	if (!stub)
+		goto unwind;
 
-        ret = wb_process_queue (process_frame, file);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "request queue processing failed");
-        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-        STACK_DESTROY (process_frame->root);
+	wb_process_queue (wb_inode);
 
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
-
-        if (process_frame) {
-                STACK_DESTROY (process_frame->root);
-        }
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        if (stub) {
+        if (stub)
                 call_stub_destroy (stub);
-        }
 
         return 0;
 }
 
 
 int32_t
-wb_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iovec *vector, int32_t count,
-              struct iatt *stbuf, struct iobref *iobref)
+wb_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        wb_local_t   *local   = NULL;
-        wb_file_t    *file    = NULL;
-        wb_request_t *request = NULL;
-        int32_t       ret     = 0;
+        GF_ASSERT (frame->local);
 
-        GF_ASSERT (frame);
+        if (op_ret == 0)
+                wb_set_inode_size (frame->local, postbuf);
 
-        local = frame->local;
-        file = local->file;
-        request = local->request;
-
-        if ((request != NULL) && (file != NULL)) {
-                wb_request_unref (request);
-
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        if (errno == ENOMEM) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                        }
-
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        }
-
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref);
+        frame->local = NULL;
 
+        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
         return 0;
 }
 
 
-static int32_t
-wb_readv_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset, uint32_t flags)
+int
+wb_ftruncate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     off_t offset, dict_t *xdata)
 {
-        STACK_WIND (frame, wb_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags);
-
+        STACK_WIND (frame, wb_ftruncate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
         return 0;
 }
 
 
-int32_t
-wb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags)
-{
-        wb_file_t    *file     = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        int32_t       ret      = -1, op_errno = 0;
-        wb_request_t *request  = NULL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, unwind,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, fd, unwind, op_errno,
-                                        EINVAL);
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
+int
+wb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
+        int32_t       op_errno     = 0;
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
+        wb_inode = wb_inode_create (this, fd->inode);
+	if (!wb_inode) {
                 op_errno = ENOMEM;
-                goto unwind;
+		goto unwind;
         }
 
-        local->file = file;
+	if (wb_fd_err (fd, this, &op_errno))
+		goto unwind;
 
-        frame->local = local;
-        if (file) {
-                stub = fop_readv_stub (frame, wb_readv_helper, fd, size,
-                                       offset, flags);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	frame->local = wb_inode;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        call_stub_destroy (stub);
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	stub = fop_ftruncate_stub (frame, wb_ftruncate_helper, fd,
+				   offset, xdata);
+	if (!stub) {
+                op_errno = ENOMEM;
+		goto unwind;
+        }
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_readv_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readv,
-                            fd, size, offset, flags);
+	if (!wb_enqueue (wb_inode, stub)) {
+                op_errno = ENOMEM;
+		goto unwind;
         }
 
+	wb_process_queue (wb_inode);
+
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+	frame->local = NULL;
+
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        if (stub)
+                call_stub_destroy (stub);
         return 0;
 }
 
 
-int32_t
-wb_ffr_bg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno)
+int
+wb_setattr_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        STACK_DESTROY (frame->root);
+        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
         return 0;
 }
 
 
-int32_t
-wb_ffr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-            int32_t op_errno)
+int
+wb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        wb_local_t *local = NULL;
-        wb_file_t  *file  = NULL;
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-        GF_ASSERT (frame);
+	wb_inode = wb_inode_ctx_get (this, loc->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        local = frame->local;
-        file = local->file;
+	stub = fop_setattr_stub (frame, wb_setattr_helper, loc, stbuf,
+                                         valid, xdata);
+	if (!stub)
+		goto unwind;
 
-        if (file != NULL) {
-                LOCK (&file->lock);
-                {
-                        if (file->op_ret == -1) {
-                                op_ret = file->op_ret;
-                                op_errno = file->op_errno;
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-                                file->op_ret = 0;
-                        }
-                }
-                UNLOCK (&file->lock);
-        }
+	wb_process_queue (wb_inode);
 
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        return 0;
+unwind:
+        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
+        if (stub)
+                call_stub_destroy (stub);
+	return 0;
+
+noqueue:
+        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
         return 0;
 }
 
 
-int32_t
-wb_flush_helper (call_frame_t *frame, xlator_t *this, fd_t *fd)
+int
+wb_fsetattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        wb_conf_t    *conf        = NULL;
-        wb_local_t   *local       = NULL;
-        wb_file_t    *file        = NULL;
-        call_frame_t *flush_frame = NULL, *process_frame = NULL;
-        int32_t       op_ret      = -1, op_errno = -1, ret = -1;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, unwind,
-                                        op_errno, EINVAL);
-
-        conf = this->private;
-
-        local = frame->local;
-        file = local->file;
-
-        LOCK (&file->lock);
-        {
-                op_ret = file->op_ret;
-                op_errno = file->op_errno;
-        }
-        UNLOCK (&file->lock);
-
-        if (local && local->request) {
-                process_frame = copy_frame (frame);
-                if (process_frame == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+}
 
-                wb_request_unref (local->request);
-        }
 
-        if (conf->flush_behind) {
-                flush_frame = copy_frame (frame);
-                if (flush_frame == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+int
+wb_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	     struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
+        call_stub_t  *stub         = NULL;
 
-                STACK_WIND (flush_frame, wb_ffr_bg_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush, fd);
-        } else {
-                STACK_WIND (frame, wb_ffr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush, fd);
-        }
+	wb_inode = wb_inode_ctx_get (this, fd->inode);
+	if (!wb_inode)
+		goto noqueue;
 
-        if (process_frame != NULL) {
-                ret = wb_process_queue (process_frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
+	stub = fop_fsetattr_stub (frame, wb_fsetattr_helper, fd, stbuf,
+				  valid, xdata);
+	if (!stub)
+		goto unwind;
 
-                STACK_DESTROY (process_frame->root);
-        }
+	if (!wb_enqueue (wb_inode, stub))
+		goto unwind;
 
-        if (conf->flush_behind) {
-                STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
-        }
+	wb_process_queue (wb_inode);
 
         return 0;
-
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
+        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        if (stub)
+                call_stub_destroy (stub);
+	return 0;
+
+noqueue:
+        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
         return 0;
 }
 
 
 int32_t
-wb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+wb_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        wb_conf_t    *conf        = NULL;
-        wb_file_t    *file        = NULL;
-        wb_local_t   *local       = NULL;
-        uint64_t      tmp_file    = 0;
-        call_stub_t  *stub        = NULL;
-        call_frame_t *flush_frame = NULL;
-        wb_request_t *request     = NULL;
-        int32_t       ret         = 0, op_errno = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, unwind,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, fd, unwind, op_errno,
-                                        EINVAL);
-
-        conf = this->private;
+        wb_inode_t   *wb_inode     = NULL;
 
+        wb_inode = wb_inode_create (this, fd->inode);
+	if (!wb_inode)
+		goto unwind;
 
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
+	if (((flags & O_RDWR) || (flags & O_WRONLY)) && (flags & O_TRUNC))
+		wb_inode->size = 0;
 
-        if (file != NULL) {
-                local = mem_get0 (this->local_pool);
-                if (local == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->create, loc, flags, mode,
+                         umask, fd, xdata);
+        return 0;
 
-                local->file = file;
+unwind:
+	STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                             NULL, NULL);
+	return 0;
+}
 
-                frame->local = local;
 
-                stub = fop_flush_stub (frame, wb_flush_helper, fd);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+int32_t
+wb_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
+{
+        wb_inode_t   *wb_inode     = NULL;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        call_stub_destroy (stub);
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        wb_inode = wb_inode_create (this, fd->inode);
+	if (!wb_inode)
+		goto unwind;
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                if (conf->flush_behind) {
-                        flush_frame = copy_frame (frame);
-                        if (flush_frame == NULL) {
-                                op_errno = ENOMEM;
-                                goto unwind;
-                        }
-
-                        STACK_UNWIND_STRICT (flush, frame, 0, 0);
-
-                        STACK_WIND (flush_frame, wb_ffr_bg_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->flush, fd);
-                } else {
-                        STACK_WIND (frame, wb_ffr_cbk, FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->flush, fd);
-                }
-        }
+	if (((flags & O_RDWR) || (flags & O_WRONLY)) && (flags & O_TRUNC))
+		wb_inode->size = 0;
 
+	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
-        return 0;
+	STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
+	return 0;
 }
 
 
-static int32_t
-wb_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf)
+int32_t
+wb_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        wb_local_t   *local   = NULL;
-        wb_file_t    *file    = NULL;
-        wb_request_t *request = NULL;
-        int32_t       ret     = -1;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        file = local->file;
-        request = local->request;
-
-        if (file != NULL) {
-                LOCK (&file->lock);
-                {
-                        if (file->op_ret == -1) {
-                                op_ret = file->op_ret;
-                                op_errno = file->op_errno;
-
-                                file->op_ret = 0;
-                        }
-                }
-                UNLOCK (&file->lock);
-
-                if (request) {
-                        wb_request_unref (request);
-                        ret = wb_process_queue (frame, file);
-                        if (ret == -1) {
-                                if (errno == ENOMEM) {
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                }
-
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "request queue processing failed");
-                        }
-                }
-
+        if (op_ret == 0) {
+                wb_inode_t *wb_inode = wb_inode_ctx_get (this, inode);
+                if (wb_inode)
+                        wb_set_inode_size (wb_inode, buf);
         }
 
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
-
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
         return 0;
 }
 
 
-static int32_t
-wb_fsync_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 int32_t datasync)
+int32_t
+wb_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+           dict_t *xdata)
 {
-        STACK_WIND (frame, wb_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, datasync);
+        STACK_WIND (frame, wb_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
         return 0;
 }
 
 
-int32_t
-wb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync)
-{
-        wb_file_t    *file     = NULL;
-        wb_local_t   *local    = NULL;
-        uint64_t      tmp_file = 0;
-        call_stub_t  *stub     = NULL;
-        wb_request_t *request  = NULL;
-        int32_t       ret      = -1, op_errno = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, this, unwind,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (frame->this->name, fd, unwind,
-                                        op_errno, EINVAL);
-
-
-        if ((!IA_ISDIR (fd->inode->ia_type))
-            && fd_ctx_get (fd, this, &tmp_file)) {
-                file = wb_file_create (this, fd, 0);
-        } else {
-                file = (wb_file_t *)(long)tmp_file;
-                if ((!IA_ISDIR (fd->inode->ia_type)) && (file == NULL)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wb_file not found for fd %p", fd);
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        local->file = file;
+int
+wb_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t    tmp      = 0;
+        wb_inode_t *wb_inode = NULL;
 
-        frame->local = local;
+        inode_ctx_del (inode, this, &tmp);
 
-        if (file) {
-                stub = fop_fsync_stub (frame, wb_fsync_helper, fd, datasync);
-                if (stub == NULL) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        wb_inode = (wb_inode_t *)(long)tmp;
 
-                request = wb_enqueue (file, stub);
-                if (request == NULL) {
-                        op_errno = ENOMEM;
-                        call_stub_destroy (stub);
-                        goto unwind;
-                }
+	if (!wb_inode)
+		return 0;
 
-                ret = wb_process_queue (frame, file);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "request queue processing failed");
-                }
-        } else {
-                STACK_WIND (frame, wb_fsync_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsync, fd, datasync);
-        }
+        GF_ASSERT (list_empty (&wb_inode->todo));
+        GF_ASSERT (list_empty (&wb_inode->liability));
+        GF_ASSERT (list_empty (&wb_inode->temptation));
 
-        return 0;
+        GF_FREE (wb_inode);
 
-unwind:
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
-int32_t
+int
 wb_release (xlator_t *this, fd_t *fd)
 {
-        uint64_t   file_ptr = 0;
-        wb_file_t *file     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+        uint64_t    tmp      = 0;
 
-        fd_ctx_get (fd, this, &file_ptr);
-        file = (wb_file_t *) (long) file_ptr;
+        fd_ctx_del (fd, this, &tmp);
 
-        if (file != NULL) {
-                LOCK (&file->lock);
-                {
-                        GF_ASSERT (list_empty (&file->request));
-                }
-                UNLOCK (&file->lock);
-
-                wb_file_destroy (file);
-        }
-
-out:
         return 0;
 }
 
@@ -2721,11 +1973,8 @@ wb_priv_dump (xlator_t *this)
 
         gf_proc_dump_write ("aggregate_size", "%d", conf->aggregate_size);
         gf_proc_dump_write ("window_size", "%d", conf->window_size);
-        gf_proc_dump_write ("disable_till", "%d", conf->disable_till);
-        gf_proc_dump_write ("enable_O_SYNC", "%d", conf->enable_O_SYNC);
         gf_proc_dump_write ("flush_behind", "%d", conf->flush_behind);
-        gf_proc_dump_write ("enable_trickling_writes", "%d",
-                            conf->enable_trickling_writes);
+        gf_proc_dump_write ("trickling_writes", "%d", conf->trickling_writes);
 
         ret = 0;
 out:
@@ -2734,131 +1983,111 @@ out:
 
 
 void
-__wb_dump_requests (struct list_head *head, char *prefix, char passive)
+__wb_dump_requests (struct list_head *head, char *prefix)
 {
         char          key[GF_DUMP_MAX_BUF_LEN]        = {0, };
         char          key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, }, flag = 0;
-        wb_request_t *request                         = NULL;
+        wb_request_t *req                             = NULL;
 
-        list_for_each_entry (request, head, list) {
-                gf_proc_dump_build_key (key, prefix, passive ? "passive-request"
-                                        : "active-request");
+        list_for_each_entry (req, head, all) {
                 gf_proc_dump_build_key (key_prefix, key,
-                                        gf_fop_list[request->fop]);
+                                        (char *)gf_fop_list[req->fop]);
 
                 gf_proc_dump_add_section(key_prefix);
 
-                gf_proc_dump_write ("request-ptr", "%p", request);
+                gf_proc_dump_write ("request-ptr", "%p", req);
 
-                gf_proc_dump_write ("refcount", "%d", request->refcount);
+                gf_proc_dump_write ("refcount", "%d", req->refcount);
 
-                if (request->fop == GF_FOP_WRITE) {
-                        flag = request->flags.write_request.stack_wound;
-                        gf_proc_dump_write ("stack_wound", "%d", flag);
+		if (list_empty (&req->todo))
+			gf_proc_dump_write ("wound", "yes");
+		else
+			gf_proc_dump_write ("wound", "no");
 
+                if (req->fop == GF_FOP_WRITE) {
                         gf_proc_dump_write ("size", "%"GF_PRI_SIZET,
-                                            request->write_size);
+                                            req->write_size);
 
                         gf_proc_dump_write ("offset", "%"PRId64,
-                                            request->stub->args.writev.off);
+                                            req->stub->args.offset);
 
-                        flag = request->flags.write_request.write_behind;
-                        gf_proc_dump_write ("write_behind", "%d", flag);
+                        flag = req->ordering.lied;
+                        gf_proc_dump_write ("lied", "%d", flag);
 
-                        flag = request->flags.write_request.got_reply;
-                        gf_proc_dump_write ("got_reply", "%d", flag);
+                        flag = req->ordering.append;
+                        gf_proc_dump_write ("append", "%d", flag);
 
-                        flag = request->flags.write_request.virgin;
-                        gf_proc_dump_write ("virgin", "%d", flag);
+                        flag = req->ordering.fulfilled;
+                        gf_proc_dump_write ("fulfilled", "%d", flag);
 
-                        flag = request->flags.write_request.flush_all;
-                        gf_proc_dump_write ("flush_all", "%d", flag);
-                } else {
-                        flag = request->flags.other_requests.marked_for_resume;
-                        gf_proc_dump_write ("marked_for_resume", "%d", flag);
+                        flag = req->ordering.go;
+                        gf_proc_dump_write ("go", "%d", flag);
                 }
         }
 }
 
 
 int
-wb_file_dump (xlator_t *this, fd_t *fd)
+wb_inode_dump (xlator_t *this, inode_t *inode)
 {
-        wb_file_t *file                            = NULL;
-        uint64_t   tmp_file                        = 0;
-        int32_t    ret                             = -1;
-        char      *path                            = NULL;
+        wb_inode_t *wb_inode                       = NULL;
+        int32_t     ret                            = -1;
+        char       *path                           = NULL;
         char       key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
+        char       uuid_str[64] = {0,};
 
-        if ((fd == NULL) || (this == NULL)) {
+        if ((inode == NULL) || (this == NULL)) {
                 ret = 0;
                 goto out;
         }
 
-        ret = fd_ctx_get (fd, this, &tmp_file);
-        if (ret == -1) {
-                ret = 0;
-                goto out;
-        }
-
-        file = (wb_file_t *)(long)tmp_file;
-        if (file == NULL) {
+        wb_inode = wb_inode_ctx_get (this, inode);
+        if (wb_inode == NULL) {
                 ret = 0;
                 goto out;
         }
 
         gf_proc_dump_build_key (key_prefix, "xlator.performance.write-behind",
-                                "file");
+                                "wb_inode");
 
         gf_proc_dump_add_section (key_prefix);
 
-        __inode_path (fd->inode, NULL, &path);
+        __inode_path (inode, NULL, &path);
         if (path != NULL) {
                 gf_proc_dump_write ("path", "%s", path);
                 GF_FREE (path);
         }
 
-        gf_proc_dump_write ("fd", "%p", fd);
-
-        gf_proc_dump_write ("disabled", "%d", file->disabled);
-
-        gf_proc_dump_write ("disable_till", "%lu", file->disable_till);
+        gf_proc_dump_write ("inode", "%p", inode);
 
-        gf_proc_dump_write ("window_conf", "%"GF_PRI_SIZET, file->window_conf);
+        gf_proc_dump_write ("window_conf", "%"GF_PRI_SIZET,
+                            wb_inode->window_conf);
 
-        gf_proc_dump_write ("window_current", "%"GF_PRI_SIZET, file->window_current);
+        gf_proc_dump_write ("window_current", "%"GF_PRI_SIZET,
+                            wb_inode->window_current);
 
-        gf_proc_dump_write ("flags", "%s", (file->flags & O_APPEND) ? "O_APPEND"
-                            : "!O_APPEND");
 
-        gf_proc_dump_write ("aggregate_current", "%"GF_PRI_SIZET, file->aggregate_current);
-
-        gf_proc_dump_write ("refcount", "%d", file->refcount);
-
-        gf_proc_dump_write ("op_ret", "%d", file->op_ret);
-
-        gf_proc_dump_write ("op_errno", "%d", file->op_errno);
-
-        LOCK (&file->lock);
+        ret = TRY_LOCK (&wb_inode->lock);
+        if (!ret)
         {
-                if (!list_empty (&file->request)) {
-                        __wb_dump_requests (&file->request, key_prefix, 0);
-                }
-
-                if (!list_empty (&file->passive_requests)) {
-                        __wb_dump_requests (&file->passive_requests, key_prefix,
-                                            1);
+                if (!list_empty (&wb_inode->all)) {
+                        __wb_dump_requests (&wb_inode->all, key_prefix);
                 }
+                UNLOCK (&wb_inode->lock);
         }
-        UNLOCK (&file->lock);
 
+        if (ret && wb_inode)
+                gf_proc_dump_write ("Unable to dump the inode information",
+                                    "(Lock acquisition failed) %p (gfid: %s)",
+                                    wb_inode,
+                                    uuid_utoa_r (inode->gfid, uuid_str));
         ret = 0;
 out:
         return ret;
 }
 
 
-int32_t
+int
 mem_acct_init (xlator_t *this)
 {
         int     ret = -1;
@@ -2892,6 +2121,14 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("flush-behind", conf->flush_behind, options, bool,
                           out);
 
+        GF_OPTION_RECONF ("trickling-writes", conf->trickling_writes, options,
+			  bool, out);
+
+        GF_OPTION_RECONF ("strict-O_DIRECT", conf->strict_O_DIRECT, options,
+			  bool, out);
+
+        GF_OPTION_RECONF ("strict-write-ordering", conf->strict_write_ordering,
+			  options, bool, out);
         ret = 0;
 out:
         return ret;
@@ -2914,7 +2151,7 @@ init (xlator_t *this)
 
         if (this->parents == NULL) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
+                        "dangling volume. check volfilex");
         }
 
         conf = GF_CALLOC (1, sizeof (*conf), gf_wb_mt_wb_conf_t);
@@ -2922,14 +2159,9 @@ init (xlator_t *this)
                 goto out;
         }
 
-        GF_OPTION_INIT("enable-O_SYNC", conf->enable_O_SYNC, bool, out);
-
         /* configure 'options aggregate-size <size>' */
         conf->aggregate_size = WB_AGGREGATE_SIZE;
 
-        GF_OPTION_INIT("disable-for-first-nbytes", conf->disable_till, size,
-                       out);
-
         /* configure 'option window-size <size>' */
         GF_OPTION_INIT ("cache-size", conf->window_size, size, out);
 
@@ -2952,24 +2184,19 @@ init (xlator_t *this)
         /* configure 'option flush-behind <on/off>' */
         GF_OPTION_INIT ("flush-behind", conf->flush_behind, bool, out);
 
-        GF_OPTION_INIT ("enable-trickling-writes", conf->enable_trickling_writes,
-                        bool, out);
+        GF_OPTION_INIT ("trickling-writes", conf->trickling_writes, bool, out);
 
-        this->local_pool = mem_pool_new (wb_local_t, 64);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
+        GF_OPTION_INIT ("strict-O_DIRECT", conf->strict_O_DIRECT, bool, out);
+
+        GF_OPTION_INIT ("strict-write-ordering", conf->strict_write_ordering,
+			bool, out);
 
         this->private = conf;
         ret = 0;
 
 out:
         if (ret) {
-                if (conf)
-                        GF_FREE (conf);
+                GF_FREE (conf);
         }
         return ret;
 }
@@ -2997,8 +2224,6 @@ out:
 
 struct xlator_fops fops = {
         .writev      = wb_writev,
-        .open        = wb_open,
-        .create      = wb_create,
         .readv       = wb_readv,
         .flush       = wb_flush,
         .fsync       = wb_fsync,
@@ -3007,17 +2232,22 @@ struct xlator_fops fops = {
         .truncate    = wb_truncate,
         .ftruncate   = wb_ftruncate,
         .setattr     = wb_setattr,
+        .fsetattr    = wb_fsetattr,
 };
 
+
 struct xlator_cbks cbks = {
+        .forget   = wb_forget,
         .release  = wb_release
 };
 
+
 struct xlator_dumpops dumpops = {
         .priv      =  wb_priv_dump,
-        .fdctx     =  wb_file_dump,
+        .inodectx  =  wb_inode_dump,
 };
 
+
 struct volume_options options[] = {
         { .key  = {"flush-behind"},
           .type = GF_OPTION_TYPE_BOOL,
@@ -3026,29 +2256,31 @@ struct volume_options options[] = {
                           "translator to perform flush in background, by "
                           "returning success (or any errors, if any of "
                           "previous  writes were failed) to application even "
-                          "before flush is sent to backend filesystem. "
+                          "before flush FOP is sent to backend filesystem. "
         },
         { .key  = {"cache-size", "window-size"},
           .type = GF_OPTION_TYPE_SIZET,
           .min  = 512 * GF_UNIT_KB,
           .max  = 1 * GF_UNIT_GB,
           .default_value = "1MB",
-          .description = "Size of the per-file write-behind buffer. "
-
-        },
-        { .key = {"disable-for-first-nbytes"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min = 0,
-          .max = 1 * GF_UNIT_MB,
-          .default_value = "0",
+          .description = "Size of the write-behind buffer for a single file "
+                         "(inode)."
         },
-        { .key = {"enable-O_SYNC"},
+        { .key = {"trickling-writes"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
         },
-        { .key = {"enable-trickling-writes"},
+        { .key = {"strict-O_DIRECT"},
           .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
+          .default_value = "off",
+          .description = "This option when set to off, ignores the "
+          "O_DIRECT flag."
+        },
+        { .key = {"strict-write-ordering"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+	  .description = "Do not let later writes overtake earlier writes even "
+	                  "if they do not overlap",
         },
         { .key = {NULL} },
 };
diff --git a/xlators/playground/Makefile.am b/xlators/playground/Makefile.am
new file mode 100644
index 000000000..e7de6b31a
--- /dev/null
+++ b/xlators/playground/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = template
+CLEANFILES =
diff --git a/xlators/playground/template/Makefile.am b/xlators/playground/template/Makefile.am
new file mode 100644
index 000000000..f26892443
--- /dev/null
+++ b/xlators/playground/template/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = src
+
diff --git a/xlators/playground/template/src/Makefile.am b/xlators/playground/template/src/Makefile.am
new file mode 100644
index 000000000..21f1c5f6b
--- /dev/null
+++ b/xlators/playground/template/src/Makefile.am
@@ -0,0 +1,16 @@
+xlator_LTLIBRARIES = template.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
+
+template_la_LDFLAGS = -module -avoid-version
+
+template_la_SOURCES = template.c
+template_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = template.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
diff --git a/xlators/playground/template/src/template.c b/xlators/playground/template/src/template.c
new file mode 100644
index 000000000..37a7794a0
--- /dev/null
+++ b/xlators/playground/template/src/template.c
@@ -0,0 +1,49 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "template.h"
+
+int32_t
+init (xlator_t *this)
+{
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "not configured with exactly one child. exiting");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        return 0;
+}
+
+void
+fini (xlator_t *this)
+{
+        return;
+}
+
+struct xlator_fops fops = {
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {NULL} },
+};
diff --git a/xlators/playground/template/src/template.h b/xlators/playground/template/src/template.h
new file mode 100644
index 000000000..d6aced501
--- /dev/null
+++ b/xlators/playground/template/src/template.h
@@ -0,0 +1,24 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __TEMPLATE_H__
+#define __TEMPLATE_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+
+#endif /* __TEMPLATE_H__ */
diff --git a/xlators/protocol/auth/addr/src/Makefile.am b/xlators/protocol/auth/addr/src/Makefile.am
index f09d1c502..426e7c2fb 100644
--- a/xlators/protocol/auth/addr/src/Makefile.am
+++ b/xlators/protocol/auth/addr/src/Makefile.am
@@ -1,12 +1,14 @@
 auth_LTLIBRARIES = addr.la
 authdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/auth
 
-addr_la_LDFLAGS = -module -avoidversion
+addr_la_LDFLAGS = -module -avoid-version
 
 addr_la_SOURCES = addr.c
 addr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/protocol/server/src \
 	-I$(top_srcdir)/rpc/rpc-lib/src/
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/auth/addr/src/addr.c b/xlators/protocol/auth/addr/src/addr.c
index 0209dd353..181d091bd 100644
--- a/xlators/protocol/auth/addr/src/addr.c
+++ b/xlators/protocol/auth/addr/src/addr.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -190,6 +181,7 @@ gf_auth (dict_t *input_params, dict_t *config_params)
                         addr_str = strtok_r (NULL, ADDR_DELIMITER, &tmp);
                 }
                 GF_FREE (addr_cpy);
+                addr_cpy = NULL;
         }
 
         if (allow_addr) {
@@ -218,22 +210,21 @@ gf_auth (dict_t *input_params, dict_t *config_params)
         }
 
 out:
-        if (addr_cpy)
-                GF_FREE (addr_cpy);
+        GF_FREE (addr_cpy);
 
         return result;
 }
 
 struct volume_options options[] = {
         { .key   = {"auth.addr.*.allow"},
-          .type  = GF_OPTION_TYPE_ANY
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
         },
         { .key   = {"auth.addr.*.reject"},
-          .type  = GF_OPTION_TYPE_ANY
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
         },
         /* Backword compatibility */
         { .key   = {"auth.ip.*.allow"},
-          .type  = GF_OPTION_TYPE_ANY
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
         },
         { .key = {NULL} }
 };
diff --git a/xlators/protocol/auth/login/src/Makefile.am b/xlators/protocol/auth/login/src/Makefile.am
index b4719d1a7..d84db91c4 100644
--- a/xlators/protocol/auth/login/src/Makefile.am
+++ b/xlators/protocol/auth/login/src/Makefile.am
@@ -1,11 +1,12 @@
 auth_LTLIBRARIES = login.la
 authdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/auth
 
-login_la_LDFLAGS = -module -avoidversion
+login_la_LDFLAGS = -module -avoid-version
 
 login_la_SOURCES = login.c
 login_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/protocol/server/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/auth/login/src/login.c b/xlators/protocol/auth/login/src/login.c
index 87ee77fcf..c2f0bf0d0 100644
--- a/xlators/protocol/auth/login/src/login.c
+++ b/xlators/protocol/auth/login/src/login.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -121,8 +112,7 @@ auth_result_t gf_auth (dict_t *input_params, dict_t *config_params)
         }
 
 out:
-        if (username_cpy)
-                GF_FREE (username_cpy);
+        GF_FREE (username_cpy);
 
         return result;
 }
diff --git a/xlators/protocol/client/src/Makefile.am b/xlators/protocol/client/src/Makefile.am
index 29c6e9dde..cf89d42da 100644
--- a/xlators/protocol/client/src/Makefile.am
+++ b/xlators/protocol/client/src/Makefile.am
@@ -2,16 +2,19 @@
 xlator_LTLIBRARIES = client.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/protocol
 
-client_la_LDFLAGS = -module -avoidversion
+client_la_LDFLAGS = -module -avoid-version
 
 client_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
-client_la_SOURCES = client.c client-helpers.c client3_1-fops.c  \
+client_la_SOURCES = client.c client-helpers.c client-rpc-fops.c  \
 	client-handshake.c client-callback.c client-lk.c
+
 noinst_HEADERS = client.h client-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)  \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src/
-\ No newline at end of file
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src/
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/client/src/client-callback.c b/xlators/protocol/client/src/client-callback.c
index 2d960ab1a..d886862f7 100644
--- a/xlators/protocol/client/src/client-callback.c
+++ b/xlators/protocol/client/src/client-callback.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -26,7 +17,7 @@
 #include "rpc-clnt.h"
 
 int
-client_cbk_null (void *data)
+client_cbk_null (struct rpc_clnt *rpc, void *mydata, void *data)
 {
         gf_log (THIS->name, GF_LOG_WARNING,
                 "this function should not be called");
@@ -34,7 +25,7 @@ client_cbk_null (void *data)
 }
 
 int
-client_cbk_fetchspec (void *data)
+client_cbk_fetchspec (struct rpc_clnt *rpc, void *mydata, void *data)
 {
         gf_log (THIS->name, GF_LOG_WARNING,
                 "this function should not be called");
@@ -42,7 +33,7 @@ client_cbk_fetchspec (void *data)
 }
 
 int
-client_cbk_ino_flush (void *data)
+client_cbk_ino_flush (struct rpc_clnt *rpc, void *mydata, void *data)
 {
         gf_log (THIS->name, GF_LOG_WARNING,
                 "this function should not be called");
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 6d6e8ebc7..85b0f757b 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -34,7 +25,8 @@
 #include "portmap-xdr.h"
 #include "rpc-common-xdr.h"
 
-extern rpc_clnt_prog_t clnt3_1_fop_prog;
+#define CLIENT_REOPEN_MAX_ATTEMPTS 1024
+extern rpc_clnt_prog_t clnt3_3_fop_prog;
 extern rpc_clnt_prog_t clnt_pmap_prog;
 
 int client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
@@ -61,7 +53,7 @@ rpc_client_ping_timer_expired (void *data)
         rpc_clnt_connection_t   *conn               = NULL;
         int                      disconnect         = 0;
         int                      transport_activity = 0;
-        struct timeval           timeout            = {0, };
+        struct timespec          timeout            = {0, };
         struct timeval           current            = {0, };
         struct rpc_clnt         *clnt               = NULL;
         xlator_t                *this               = NULL;
@@ -109,7 +101,7 @@ rpc_client_ping_timer_expired (void *data)
                                 "ping timer expired but transport activity "
                                 "detected - not bailing transport");
                         timeout.tv_sec = conf->opt.ping_timeout;
-                        timeout.tv_usec = 0;
+                        timeout.tv_nsec = 0;
 
                         conn->ping_timer =
                                 gf_timer_call_after (this->ctx, timeout,
@@ -148,7 +140,7 @@ client_start_ping (void *data)
         clnt_conf_t             *conf        = NULL;
         rpc_clnt_connection_t   *conn        = NULL;
         int32_t                  ret         = -1;
-        struct timeval           timeout     = {0, };
+        struct timespec          timeout     = {0, };
         call_frame_t            *frame       = NULL;
         int                      frame_count = 0;
 
@@ -204,7 +196,7 @@ client_start_ping (void *data)
                 }
 
                 timeout.tv_sec = conf->opt.ping_timeout;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 conn->ping_timer =
                         gf_timer_call_after (this->ctx, timeout,
@@ -227,14 +219,14 @@ client_start_ping (void *data)
         ret = client_submit_request (this, NULL, frame, conf->handshake,
                                      GF_HNDSK_PING, client_ping_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL, (xdrproc_t)NULL);
-        if (ret)
-                goto fail;
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to start ping timer");
+        }
 
         return;
-fail:
-        gf_log (THIS->name, GF_LOG_ERROR,
-                "failed to start ping timer");
 
+fail:
         if (frame) {
                 STACK_DESTROY (frame->root);
         }
@@ -249,42 +241,47 @@ client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
 {
         xlator_t              *this    = NULL;
         rpc_clnt_connection_t *conn    = NULL;
-        struct timeval         timeout = {0, };
+        struct timespec        timeout = {0, };
         call_frame_t          *frame   = NULL;
         clnt_conf_t           *conf    = NULL;
 
         if (!myframe) {
-                gf_log (THIS->name, GF_LOG_WARNING, "frame with the request is NULL");
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "frame with the request is NULL");
                 goto out;
         }
         frame = myframe;
         this = frame->this;
         if (!this || !this->private) {
-                gf_log (THIS->name, GF_LOG_WARNING, "xlator private is not set");
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "xlator private is not set");
                 goto out;
         }
 
         conf = this->private;
         conn = &conf->rpc->conn;
 
-        if (req->rpc_status == -1) {
-                if (conn->ping_timer != NULL) {
-                        gf_log (this->name, GF_LOG_WARNING, "socket or ib"
-                                " related error");
-                        gf_timer_call_cancel (this->ctx, conn->ping_timer);
-                        conn->ping_timer = NULL;
-                } else {
-                        /* timer expired and transport bailed out */
-                        gf_log (this->name, GF_LOG_WARNING, "timer must have "
-                                "expired");
-                }
-                goto out;
-        }
-
         pthread_mutex_lock (&conn->lock);
         {
+                if (req->rpc_status == -1) {
+                        if (conn->ping_timer != NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "socket or ib related error");
+                                gf_timer_call_cancel (this->ctx,
+                                                      conn->ping_timer);
+                                conn->ping_timer = NULL;
+                        } else {
+                                /* timer expired and transport bailed out */
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "timer must have expired");
+                        }
+
+                        goto unlock;
+                }
+
+
                 timeout.tv_sec  = conf->opt.ping_timeout;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 gf_timer_call_cancel (this->ctx,
                                       conn->ping_timer);
@@ -297,6 +294,7 @@ client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         gf_log (this->name, GF_LOG_WARNING,
                                 "failed to set the ping timer");
         }
+unlock:
         pthread_mutex_unlock (&conn->lock);
 out:
         if (frame)
@@ -346,11 +344,11 @@ client3_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
 out:
-        STACK_UNWIND_STRICT (getspec, frame, rsp.op_ret, rsp.op_errno, rsp.spec);
+        CLIENT_STACK_UNWIND (getspec, frame, rsp.op_ret, rsp.op_errno,
+                             rsp.spec);
 
         /* Don't use 'GF_FREE', this is allocated by libc */
-        if (rsp.spec)
-                free (rsp.spec);
+        free (rsp.spec);
 
         return 0;
 }
@@ -376,13 +374,14 @@ int32_t client3_getspec (call_frame_t *frame, xlator_t *this, void *data)
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gf_getspec_req);
 
-        if (ret)
-                goto unwind;
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to send the request");
+        }
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the request");
-        STACK_UNWIND_STRICT (getspec, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (getspec, frame, -1, op_errno, NULL);
         return 0;
 
 }
@@ -404,6 +403,28 @@ client_notify_parents_child_up (xlator_t *this)
 }
 
 int
+clnt_fd_lk_reacquire_failed (xlator_t *this, clnt_fd_ctx_t *fdctx,
+                             clnt_conf_t *conf)
+{
+        int      ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx->remote_fd     = -1;
+                fdctx->lk_heal_state = GF_LK_HEAL_DONE;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
                            int count, void *myframe)
 {
@@ -425,7 +446,7 @@ client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
                 gf_log (fr->this->name, GF_LOG_WARNING,
                         "xdr decoding failed");
         else
-                gf_log (fr->this->name, GF_LOG_DEBUG,
+                gf_log (fr->this->name, GF_LOG_INFO,
                         "Server lk version = %d", rsp.lk_ver);
 
         ret = 0;
@@ -436,6 +457,7 @@ out:
         return ret;
 }
 
+//TODO: Check for all released fdctx and destroy them
 int
 client_set_lk_version (xlator_t *this)
 {
@@ -443,17 +465,31 @@ client_set_lk_version (xlator_t *this)
         clnt_conf_t        *conf     = NULL;
         call_frame_t       *frame    = NULL;
         gf_set_lk_ver_req   req      = {0, };
+        char               *process_uuid = NULL;
+
+        GF_VALIDATE_OR_GOTO ("client", this, err);
 
         conf = (clnt_conf_t *) this->private;
 
         req.lk_ver = client_get_lk_ver (conf);
-        req.uid    = this->ctx->process_uuid;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Sending SET_LK_VERSION");
+        ret = dict_get_str (this->options, "process-uuid", &process_uuid);
+        if (!process_uuid) {
+                ret = -1;
+                goto err;
+        }
+        req.uid = gf_strdup (process_uuid);
+        if (!req.uid) {
+                ret = -1;
+                goto err;
+        }
 
         frame = create_frame (this, this->ctx->pool);
-        if (!frame)
+        if (!frame) {
+                ret = -1;
                 goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sending SET_LK_VERSION");
 
         ret = client_submit_request (this, &req, frame,
                                      conf->handshake,
@@ -462,9 +498,11 @@ client_set_lk_version (xlator_t *this)
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gf_set_lk_ver_req);
 out:
-        if (ret < 0)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Failed to send SET_LK_VERSION to server");
+        GF_FREE (req.uid);
+        return ret;
+err:
+        gf_log (this->name, GF_LOG_WARNING,
+                "Failed to send SET_LK_VERSION to server");
 
         return ret;
 }
@@ -520,7 +558,6 @@ clnt_fd_lk_local_unref (xlator_t *this, clnt_fd_lk_local_t *local)
                 LOCK_DESTROY (&local->lock);
                 GF_FREE (local);
         }
-        ref = 0;
 out:
         return ref;
 }
@@ -554,7 +591,6 @@ clnt_mark_fd_bad (clnt_conf_t *conf, clnt_fd_ctx_t *fdctx)
         pthread_mutex_unlock (&conf->lock);
 }
 
-// call decrement_reopen_fd_count
 int
 clnt_release_reopen_fd_cbk (struct rpc_req *req, struct iovec *iov,
                             int count, void *myframe)
@@ -569,9 +605,9 @@ clnt_release_reopen_fd_cbk (struct rpc_req *req, struct iovec *iov,
         fdctx  = (clnt_fd_ctx_t *) frame->local;
         conf   = (clnt_conf_t *) this->private;
 
-        clnt_mark_fd_bad (conf, fdctx);
+        clnt_fd_lk_reacquire_failed (this, fdctx, conf);
 
-        decrement_reopen_fd_count (this, conf);
+        fdctx->reopen_done (fdctx, this);
 
         frame->local = NULL;
         STACK_DESTROY (frame->root);
@@ -589,7 +625,7 @@ clnt_release_reopen_fd (xlator_t *this, clnt_fd_ctx_t *fdctx)
 
         conf = (clnt_conf_t *) this->private;
 
-        frame  = create_frame (THIS, THIS->ctx->pool);
+        frame  = create_frame (this, this->ctx->pool);
         if (!frame)
                 goto out;
 
@@ -601,25 +637,64 @@ clnt_release_reopen_fd (xlator_t *this, clnt_fd_ctx_t *fdctx)
                                         clnt_release_reopen_fd_cbk, NULL,
                                         NULL, 0, NULL, 0, NULL,
                                         (xdrproc_t)xdr_gfs3_releasedir_req);
-out:
+        return 0;
+ out:
         if (ret) {
-                decrement_reopen_fd_count (this, conf);
-                clnt_mark_fd_bad (conf, fdctx);
+                clnt_fd_lk_reacquire_failed (this, fdctx, conf);
+                fdctx->reopen_done (fdctx, this);
                 if (frame) {
                         frame->local = NULL;
                         STACK_DESTROY (frame->root);
                 }
         }
-
         return 0;
 }
 
 int
+clnt_reacquire_lock_error (xlator_t *this, clnt_fd_ctx_t *fdctx,
+                           clnt_conf_t *conf)
+{
+        int32_t   ret       = -1;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        clnt_release_reopen_fd (this, fdctx);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+gf_boolean_t
+clnt_fd_lk_local_error_status (xlator_t *this,
+                               clnt_fd_lk_local_t *local)
+{
+        gf_boolean_t error = _gf_false;
+
+        LOCK (&local->lock);
+        {
+                error = local->error;
+        }
+        UNLOCK (&local->lock);
+
+        return error;
+}
+
+int
 clnt_fd_lk_local_mark_error (xlator_t *this,
                              clnt_fd_lk_local_t *local)
 {
+        int32_t       ret   = -1;
+        clnt_conf_t  *conf  = NULL;
         gf_boolean_t  error = _gf_false;
 
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        conf = (clnt_conf_t *) this->private;
+
         LOCK (&local->lock);
         {
                 error        = local->error;
@@ -627,30 +702,30 @@ clnt_fd_lk_local_mark_error (xlator_t *this,
         }
         UNLOCK (&local->lock);
 
-        if (error)
-                clnt_release_reopen_fd (this, local->fdctx);
-
-        return 0;
+        if (!error)
+                clnt_reacquire_lock_error (this, local->fdctx, conf);
+        ret = 0;
+out:
+        return ret;
 }
 
-// Also, I think in reopen_cbk, the fdctx is added to
-// saved_fd list.. avoid that, may cause a problem
-// Reason: While the locks on the fd are reacquired, a release
-// fop may be received by the client-protocol translator
-// which will free the fdctx datastructure.
 int
 client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
                            int count, void *myframe)
 {
         int32_t             ret        = -1;
         xlator_t           *this       = NULL;
-        gf_common_rsp       rsp        = {0,};
+        gfs3_lk_rsp         rsp        = {0,};
         call_frame_t       *frame      = NULL;
+        clnt_conf_t        *conf       = NULL;
+        clnt_fd_ctx_t      *fdctx      = NULL;
         clnt_fd_lk_local_t *local      = NULL;
+        struct gf_flock     lock       = {0,};
 
         frame = (call_frame_t *) myframe;
         this  = frame->this;
         local = (clnt_fd_lk_local_t *) frame->local;
+        conf  = (clnt_conf_t *) this->private;
 
         if (req->rpc_status == -1) {
                 gf_log ("client", GF_LOG_WARNING,
@@ -658,7 +733,7 @@ client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lk_rsp);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
                 goto out;
@@ -670,15 +745,34 @@ client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-        // TODO: Add more info to log.
-        gf_log (this->name, GF_LOG_DEBUG, "Reacquired lock");
+        fdctx = local->fdctx;
+
+        gf_proto_flock_to_flock (&rsp.flock, &lock);
+
+        gf_log (this->name, GF_LOG_DEBUG, "%s type lock reacquired on file "
+                "with gfid %s from %"PRIu64 " to %"PRIu64,
+                get_lk_type (lock.l_type), uuid_utoa (fdctx->gfid),
+                lock.l_start, lock.l_start + lock.l_len);
+
+        if (!clnt_fd_lk_local_error_status (this, local) &&
+            clnt_fd_lk_local_unref (this, local) == 0) {
+                pthread_mutex_lock (&conf->lock);
+                {
+                        fdctx->lk_heal_state = GF_LK_HEAL_DONE;
+                }
+                pthread_mutex_unlock (&conf->lock);
+
+                fdctx->reopen_done (fdctx, this);
+        }
 
         ret = 0;
 out:
-        if (ret < 0)
+        if (ret < 0) {
                 clnt_fd_lk_local_mark_error (this, local);
 
-        (void) clnt_fd_lk_local_unref (this, local);
+                clnt_fd_lk_local_unref (this, local);
+        }
+
         frame->local = NULL;
         STACK_DESTROY (frame->root);
 
@@ -704,7 +798,10 @@ _client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
 
         local = clnt_fd_lk_local_create (fdctx);
         if (!local) {
-                clnt_release_reopen_fd (this, fdctx);
+                gf_log (this->name, GF_LOG_WARNING, "clnt_fd_lk_local_create "
+                        "failed, aborting reacquring of locks on %s.",
+                        uuid_utoa (fdctx->gfid));
+                clnt_reacquire_lock_error (this, fdctx, conf);
                 goto out;
         }
 
@@ -712,7 +809,9 @@ _client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
                 memcpy (&flock, &fd_lk->user_flock,
                         sizeof (struct gf_flock));
 
-                ret = client_cmd_to_gf_cmd (fd_lk->cmd, &gf_cmd);
+                /* Always send F_SETLK even if the cmd was F_SETLKW */
+                /* to avoid frame being blocked if lock cannot be granted. */
+                ret = client_cmd_to_gf_cmd (F_SETLK, &gf_cmd);
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "client_cmd_to_gf_cmd failed, "
@@ -727,9 +826,9 @@ _client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
                 (void) gf_proto_flock_from_flock (&req.flock,
                                                   &flock);
 
-                memcpy (req.gfid, fdctx->inode->gfid, 16);
+                memcpy (req.gfid, fdctx->gfid, 16);
 
-                frame = create_frame (THIS, THIS->ctx->pool);
+                frame = create_frame (this, this->ctx->pool);
                 if (!frame) {
                         ret = -1;
                         break;
@@ -743,24 +842,17 @@ _client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
                                              client_reacquire_lock_cbk,
                                              NULL, NULL, 0, NULL, 0, NULL,
                                              (xdrproc_t)xdr_gfs3_lk_req);
-                if (ret)
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "reacquiring locks failed on file with gfid %s",
+                                uuid_utoa (fdctx->gfid));
                         break;
+                }
 
                 ret   = 0;
                 frame = NULL;
         }
 
-        if (ret) {
-                clnt_fd_lk_local_mark_error (this, local);
-
-                if (frame) {
-                        if (frame->local) {
-                                clnt_fd_lk_local_unref (this, frame->local);
-                                frame->local = NULL;
-                        }
-                        STACK_DESTROY (frame->root);
-                }
-        }
         if (local)
                 (void) clnt_fd_lk_local_unref (this, local);
 out:
@@ -773,33 +865,87 @@ client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
         int32_t          ret       = -1;
         fd_lk_ctx_t     *lk_ctx    = NULL;
 
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+
         if (client_fd_lk_list_empty (fdctx->lk_ctx, _gf_false)) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "fd lock list is empty");
-                decrement_reopen_fd_count (this, (clnt_conf_t *)this->private);
-                ret = 0;
-                goto out;
+                fdctx->reopen_done (fdctx, this);
+        } else {
+                lk_ctx = fdctx->lk_ctx;
+
+                LOCK (&lk_ctx->lock);
+                {
+                        (void) _client_reacquire_lock (this, fdctx);
+                }
+                UNLOCK (&lk_ctx->lock);
         }
+        ret = 0;
+out:
+        return ret;
+}
 
-        lk_ctx = fdctx->lk_ctx;
+void
+client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        gf_log_callingfn (this->name, GF_LOG_WARNING,
+                          "This function should never be called");
+}
 
-        LOCK (&lk_ctx->lock);
+void
+client_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        clnt_conf_t  *conf    = NULL;
+        gf_boolean_t destroy  = _gf_false;
+
+        conf = this->private;
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx->reopen_attempts = 0;
+                if (!fdctx->released)
+                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
+                else
+                        destroy = _gf_true;
+                fdctx->reopen_done = client_default_reopen_done;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        if (destroy)
+                client_fdctx_destroy (this, fdctx);
+}
+
+void
+client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        clnt_conf_t  *conf    = NULL;
+        uint64_t     fd_count = 0;
+
+        conf = this->private;
+
+        LOCK (&conf->rec_lock);
         {
-                ret = _client_reacquire_lock (this, fdctx);
+                fd_count = --(conf->reopen_fd_count);
+        }
+        UNLOCK (&conf->rec_lock);
+
+        client_reopen_done (fdctx, this);
+        if (fd_count == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "last fd open'd/lock-self-heal'd - notifying CHILD-UP");
+                client_set_lk_version (this);
+                client_notify_parents_child_up (this);
         }
-        UNLOCK (&lk_ctx->lock);
-out:
-        return ret;
 }
 
 int
-client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void           *myframe)
 {
         int32_t        ret                   = -1;
         gfs3_open_rsp  rsp                   = {0,};
-        int            attempt_lock_recovery = _gf_false;
-        uint64_t       fd_count              = 0;
+        gf_boolean_t   attempt_lock_recovery = _gf_false;
         clnt_local_t  *local                 = NULL;
         clnt_conf_t   *conf                  = NULL;
         clnt_fd_ctx_t *fdctx                 = NULL;
@@ -807,12 +953,10 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
         xlator_t      *this                  = NULL;
 
         frame = myframe;
-        if (!frame || !frame->this)
-                goto out;
-
         this  = frame->this;
+        conf  = this->private;
         local = frame->local;
-        conf  = frame->this->private;
+        fdctx = local->fdctx;
 
         if (-1 == req->rpc_status) {
                 gf_log (frame->this->name, GF_LOG_WARNING,
@@ -845,49 +989,41 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-        fdctx = local->fdctx;
-
-        if (!fdctx) {
-                gf_log (frame->this->name, GF_LOG_WARNING, "fdctx not found");
-                ret = -1;
-                goto out;
-        }
-
         pthread_mutex_lock (&conf->lock);
         {
                 fdctx->remote_fd = rsp.fd;
                 if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        if (!client_fd_lk_list_empty (fdctx->lk_ctx, _gf_false))
+                        if (conf->lk_heal &&
+                            !client_fd_lk_list_empty (fdctx->lk_ctx,
+                                                      _gf_false)) {
                                 attempt_lock_recovery = _gf_true;
-                        fdctx = NULL;
+                                fdctx->lk_heal_state  = GF_LK_HEAL_IN_PROGRESS;
+                        }
                 }
         }
         pthread_mutex_unlock (&conf->lock);
 
         ret = 0;
 
-        if (conf->lk_heal && attempt_lock_recovery) {
-                /* Delay decrement the reopen fd count untill all the
+        if (attempt_lock_recovery) {
+                /* Delay decrementing the reopen fd count untill all the
                    locks corresponding to this fd are acquired.*/
-                gf_log (frame->this->name, GF_LOG_WARNING, "acquiring locks on "
-                        "%s", local->loc.path);
+                gf_log (this->name, GF_LOG_DEBUG, "acquiring locks "
+                        "on %s", local->loc.path);
                 ret = client_reacquire_lock (frame->this, local->fdctx);
-        } else {
-                fd_count = decrement_reopen_fd_count (frame->this, conf);
+                if (ret) {
+                        clnt_reacquire_lock_error (this, local->fdctx, conf);
+                        gf_log (this->name, GF_LOG_WARNING, "acquiring locks "
+                                "failed on %s", local->loc.path);
+                }
         }
-out:
 
-        if (fdctx)
-                client_fdctx_destroy (this, fdctx);
-
-        if ((ret < 0) && frame && frame->this && conf)
-                decrement_reopen_fd_count (frame->this, conf);
+out:
+        if (!attempt_lock_recovery)
+                fdctx->reopen_done (fdctx, this);
 
-        if (frame) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-        }
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
 
         client_local_wipe (local);
 
@@ -895,7 +1031,7 @@ out:
 }
 
 int
-client3_1_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void           *myframe)
 {
         int32_t        ret   = -1;
@@ -906,11 +1042,10 @@ client3_1_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         call_frame_t  *frame = NULL;
 
         frame = myframe;
-        if (!frame || !frame->this)
-                goto out;
+        local = frame->local;
+        fdctx = local->fdctx;
+        conf  = frame->this->private;
 
-        local        = frame->local;
-        conf         = frame->this->private;
 
         if (-1 == req->rpc_status) {
                 gf_log (frame->this->name, GF_LOG_WARNING,
@@ -943,78 +1078,44 @@ client3_1_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-        fdctx = local->fdctx;
-        if (!fdctx) {
-                gf_log (frame->this->name, GF_LOG_WARNING, "fdctx not found");
-                ret = -1;
-                goto out;
-        }
-
         pthread_mutex_lock (&conf->lock);
         {
                 fdctx->remote_fd = rsp.fd;
-
-                if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        fdctx = NULL;
-                }
         }
         pthread_mutex_unlock (&conf->lock);
 
-        decrement_reopen_fd_count (frame->this, conf);
-        ret = 0;
-
 out:
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
-
-        if ((ret < 0) && frame && frame->this && conf)
-                decrement_reopen_fd_count (frame->this, conf);
-
-        if (frame) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-        }
+        fdctx->reopen_done (fdctx, frame->this);
 
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
         client_local_wipe (local);
 
         return 0;
 }
 
-int
-protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx)
+static int
+protocol_client_reopendir (clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
         int               ret   = -1;
         gfs3_opendir_req  req   = {{0,},};
         clnt_local_t     *local = NULL;
-        inode_t          *inode = NULL;
-        char             *path  = NULL;
         call_frame_t     *frame = NULL;
         clnt_conf_t      *conf  = NULL;
 
-        if (!this || !fdctx)
-                goto out;
-
-        inode = fdctx->inode;
         conf = this->private;
 
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "couldn't build path from inode %s",
-                        uuid_utoa (inode->gfid));
-                goto out;
-        }
-
         local = mem_get0 (this->local_pool);
         if (!local) {
                 ret = -1;
                 goto out;
         }
-
         local->fdctx    = fdctx;
-        local->loc.path = path;
-        path            = NULL;
+
+        uuid_copy (local->loc.gfid, fdctx->gfid);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
 
         frame = create_frame (this, this->ctx->pool);
         if (!frame) {
@@ -1022,27 +1123,26 @@ protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx)
                 goto out;
         }
 
-        memcpy (req.gfid, inode->gfid, 16);
+        memcpy (req.gfid, fdctx->gfid, 16);
 
         gf_log (frame->this->name, GF_LOG_DEBUG,
                 "attempting reopen on %s", local->loc.path);
 
-        frame->local = local; local = NULL;
+        frame->local = local;
 
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_OPENDIR,
-                                     client3_1_reopendir_cbk, NULL,
+                                     client3_3_reopendir_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_opendir_req);
-        if (ret)
-                goto out;
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to send the re-opendir request");
+        }
 
-        return ret;
+        return 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_ERROR,
-                "failed to send the re-opendir request");
-
         if (frame) {
                 frame->local = NULL;
                 STACK_DESTROY (frame->root);
@@ -1051,41 +1151,23 @@ out:
         if (local)
                 client_local_wipe (local);
 
-        if (path)
-                GF_FREE (path);
-        if ((ret < 0) && this && conf) {
-                decrement_reopen_fd_count (this, conf);
-        }
+        fdctx->reopen_done (fdctx, this);
 
         return 0;
 
 }
 
-int
-protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx)
+static int
+protocol_client_reopenfile (clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
         int            ret   = -1;
         gfs3_open_req  req   = {{0,},};
         clnt_local_t  *local = NULL;
-        inode_t       *inode = NULL;
-        char          *path  = NULL;
         call_frame_t  *frame = NULL;
         clnt_conf_t   *conf  = NULL;
 
-        if (!this || !fdctx)
-                goto out;
-
-        inode = fdctx->inode;
         conf  = this->private;
 
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "couldn't build path from inode %s",
-                        uuid_utoa (inode->gfid));
-                goto out;
-        }
-
         frame = create_frame (this, this->ctx->pool);
         if (!frame) {
                 ret = -1;
@@ -1099,31 +1181,32 @@ protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx)
         }
 
         local->fdctx    = fdctx;
-        local->loc.path = path;
-        path            = NULL;
+        uuid_copy (local->loc.gfid, fdctx->gfid);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
+
         frame->local    = local;
 
-        memcpy (req.gfid, inode->gfid, 16);
+        memcpy (req.gfid, fdctx->gfid, 16);
         req.flags    = gf_flags_from_flags (fdctx->flags);
-        req.wbflags  = fdctx->wbflags;
+        req.flags    = req.flags & (~(O_TRUNC|O_CREAT|O_EXCL));
 
         gf_log (frame->this->name, GF_LOG_DEBUG,
                 "attempting reopen on %s", local->loc.path);
 
-        local = NULL;
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPEN, client3_1_reopen_cbk, NULL,
+                                     GFS3_OP_OPEN, client3_3_reopen_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_open_req);
-        if (ret)
-                goto out;
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to send the re-open request");
+        }
 
-        return ret;
+        return 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_ERROR,
-                "failed to send the re-open request");
-
         if (frame) {
                 frame->local = NULL;
                 STACK_DESTROY (frame->root);
@@ -1132,17 +1215,65 @@ out:
         if (local)
                 client_local_wipe (local);
 
-        if (path)
-                GF_FREE (path);
-
-        if ((ret < 0) && this && conf) {
-                decrement_reopen_fd_count (this, conf);
-        }
+        fdctx->reopen_done (fdctx, this);
 
         return 0;
 
 }
 
+static void
+protocol_client_reopen (clnt_fd_ctx_t *fdctx, xlator_t *this)
+{
+        if (fdctx->is_dir)
+                protocol_client_reopendir (fdctx, this);
+        else
+                protocol_client_reopenfile (fdctx, this);
+}
+
+gf_boolean_t
+__is_fd_reopen_in_progress (clnt_fd_ctx_t *fdctx)
+{
+        if (fdctx->reopen_done == client_default_reopen_done)
+                return _gf_false;
+        return _gf_true;
+}
+
+void
+client_attempt_reopen (fd_t *fd, xlator_t *this)
+{
+        clnt_conf_t   *conf  = NULL;
+        clnt_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t  reopen = _gf_false;
+
+        if (!fd || !this)
+                goto out;
+
+        conf = this->private;
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx = this_fd_get_ctx (fd, this);
+                if (!fdctx)
+                        goto unlock;
+                if (__is_fd_reopen_in_progress (fdctx))
+                        goto unlock;
+                if (fdctx->remote_fd != -1)
+                        goto unlock;
+
+                if (fdctx->reopen_attempts == CLIENT_REOPEN_MAX_ATTEMPTS) {
+                        reopen = _gf_true;
+                        fdctx->reopen_done = client_reopen_done;
+                        list_del_init (&fdctx->sfd_pos);
+                } else {
+                        fdctx->reopen_attempts++;
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&conf->lock);
+        if (reopen)
+                protocol_client_reopen (fdctx, this);
+out:
+        return;
+}
 
 int
 client_post_handshake (call_frame_t *frame, xlator_t *this)
@@ -1167,6 +1298,7 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
                         if (fdctx->remote_fd != -1)
                                 continue;
 
+                        fdctx->reopen_done = client_child_up_reopen_done;
                         list_del_init (&fdctx->sfd_pos);
                         list_add_tail (&fdctx->sfd_pos, &reopen_head);
                         count++;
@@ -1185,14 +1317,11 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
                 list_for_each_entry_safe (fdctx, tmp, &reopen_head, sfd_pos) {
                         list_del_init (&fdctx->sfd_pos);
 
-                        if (fdctx->is_dir)
-                                protocol_client_reopendir (this, fdctx);
-                        else
-                                protocol_client_reopen (this, fdctx);
+                        protocol_client_reopen (fdctx, this);
                 }
         } else {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "no fds to open - notifying all parents child up");
+                        "No fds to open - notifying all parents child up");
                 client_set_lk_version (this);
                 client_notify_parents_child_up (this);
         }
@@ -1274,8 +1403,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
                         remote_error ? remote_error : strerror (op_errno));
                 errno = op_errno;
                 if (remote_error &&
-                   (strncmp ("Authentication failed",remote_error,
-                             sizeof (remote_error)) == 0)) {
+                    (strcmp ("Authentication failed", remote_error) == 0)) {
                         auth_fail = _gf_true;
                         op_ret = 0;
                 }
@@ -1304,7 +1432,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
                 goto out;
         }
 
-        gf_log (this->name, GF_LOG_INFO, "clnt-lk-version = %d, "
+        gf_log (this->name, GF_LOG_DEBUG, "clnt-lk-version = %d, "
                 "server-lk-version = %d", client_get_lk_ver (conf), lk_ver);
         /* TODO: currently setpeer path is broken */
         /*
@@ -1330,7 +1458,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
 
         gf_log (this->name, GF_LOG_INFO,
                 "Connected to %s, attached to remote volume '%s'.",
-                conf->rpc->conn.trans->peerinfo.identifier,
+                conf->rpc->conn.name,
                 remote_subvol);
 
         rpc_clnt_set_connected (&conf->rpc->conn);
@@ -1342,13 +1470,18 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
         conf->need_different_port = 0;
 
         if (lk_ver != client_get_lk_ver (conf)) {
+                gf_log (this->name, GF_LOG_INFO, "Server and Client "
+                        "lk-version numbers are not same, reopening the fds");
                 client_mark_fd_bad (this);
                 client_post_handshake (frame, frame->this);
         } else {
                 /*TODO: Traverse the saved fd list, and send
                   release to the server on fd's that were closed
                   during grace period */
-                ;
+                gf_log (this->name, GF_LOG_INFO, "Server and Client "
+                        "lk-version numbers are same, no need to "
+                        "reopen the fds");
+                client_notify_parents_child_up (frame->this);
         }
 
 out:
@@ -1378,8 +1511,7 @@ out:
                 ret = 0;
         }
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+        free (rsp.dict.dict_val);
 
         STACK_DESTROY (frame->root);
 
@@ -1398,6 +1530,7 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
         char             *process_uuid_xl = NULL;
         clnt_conf_t      *conf            = NULL;
         dict_t           *options         = NULL;
+        char             counter_str[32]  = {0};
 
         options = this->options;
         conf    = this->private;
@@ -1423,13 +1556,24 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
                 }
         }
 
-        /* With multiple graphs possible in the same process, we need a
+        /* When lock-heal is enabled:
+         * With multiple graphs possible in the same process, we need a
            field to bring the uniqueness. Graph-ID should be enough to get the
-           job done
+           job done.
+         * When lock-heal is disabled, connection-id should always be unique so
+         * that server never gets to reuse the previous connection resources
+         * so it cleans up the resources on every disconnect. Otherwise
+         * it may lead to stale resources, i.e. leaked file desciptors,
+         * inode/entry locks
         */
-        ret = gf_asprintf (&process_uuid_xl, "%s-%s-%d",
+        if (!conf->lk_heal) {
+                snprintf (counter_str, sizeof (counter_str),
+                          "-%"PRIu64, conf->setvol_count);
+                conf->setvol_count++;
+        }
+        ret = gf_asprintf (&process_uuid_xl, "%s-%s-%d%s",
                            this->ctx->process_uuid, this->name,
-                           this->graph->id);
+                           this->graph->id, counter_str);
         if (-1 == ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "asprintf failed while setting process_uuid");
@@ -1474,13 +1618,14 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
                         client_get_lk_ver (conf));
         }
 
-        req.dict.dict_len = dict_serialized_length (options);
-        if (req.dict.dict_len < 0) {
+        ret = dict_serialized_length (options);
+        if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to get serialized length of dict");
                 ret = -1;
                 goto fail;
         }
+        req.dict.dict_len = ret;
         req.dict.dict_val = GF_CALLOC (1, req.dict.dict_len,
                                        gf_client_mt_clnt_req_buf_t);
         ret = dict_serialize (options, req.dict.dict_val);
@@ -1500,8 +1645,7 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
                                      (xdrproc_t)xdr_gf_setvolume_req);
 
 fail:
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
         return ret;
 }
@@ -1524,9 +1668,9 @@ select_server_supported_programs (xlator_t *this, gf_prog_detail *prog)
 
         while (trav) {
                 /* Select 'programs' */
-                if ((clnt3_1_fop_prog.prognum == trav->prognum) &&
-                    (clnt3_1_fop_prog.progver == trav->progver)) {
-                        conf->fops = &clnt3_1_fop_prog;
+                if ((clnt3_3_fop_prog.prognum == trav->prognum) &&
+                    (clnt3_3_fop_prog.progver == trav->progver)) {
+                        conf->fops = &clnt3_3_fop_prog;
                         gf_log (this->name, GF_LOG_INFO,
                                 "Using Program %s, Num (%"PRId64"), "
                                 "Version (%"PRId64")",
@@ -1611,16 +1755,21 @@ client_query_portmap_cbk (struct rpc_req *req, struct iovec *iov, int count, voi
                 ret = -1;
                 gf_log (this->name, ((!conf->portmap_err_logged) ?
                                      GF_LOG_ERROR : GF_LOG_DEBUG),
-                        "failed to get the port number for remote subvolume");
+                        "failed to get the port number for remote subvolume. "
+                        "Please run 'gluster volume status' on server to see "
+                        "if brick process is running.");
                 conf->portmap_err_logged = 1;
                 goto out;
         }
 
         conf->portmap_err_logged = 0;
+        conf->disconnect_err_logged = 0;
 
         config.remote_port = rsp.port;
         rpc_clnt_reconfig (conf->rpc, &config);
+
         conf->skip_notify = 1;
+	conf->quick_reconnect = 1;
 
 out:
         if (frame)
@@ -1630,7 +1779,6 @@ out:
                 /* Need this to connect the same transport on different port */
                 /* ie, glusterd to glusterfsd */
                 rpc_transport_disconnect (conf->rpc->conn.trans);
-                rpc_clnt_reconnect (conf->rpc->conn.trans);
         }
 
         return ret;
diff --git a/xlators/protocol/client/src/client-helpers.c b/xlators/protocol/client/src/client-helpers.c
index 8cf83fe30..5d9f00fdc 100644
--- a/xlators/protocol/client/src/client-helpers.c
+++ b/xlators/protocol/client/src/client-helpers.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -25,7 +16,6 @@
 #include "client.h"
 #include "fd.h"
 
-
 int
 client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t try_lock)
 {
@@ -132,6 +122,7 @@ client_local_wipe (clnt_local_t *local)
 {
         if (local) {
                 loc_wipe (&local->loc);
+                loc_wipe (&local->loc2);
 
                 if (local->fd) {
                         fd_unref (local->fd);
@@ -141,7 +132,9 @@ client_local_wipe (clnt_local_t *local)
                         iobref_unref (local->iobref);
                 }
 
-                 mem_put (local);
+                GF_FREE (local->name);
+
+                mem_put (local);
         }
 
         return 0;
@@ -282,3 +275,75 @@ clnt_readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
 
         return 0;
 }
+
+int
+client_get_remote_fd (xlator_t *this, fd_t *fd, int flags, int64_t *remote_fd)
+{
+        clnt_fd_ctx_t *fdctx    = NULL;
+        clnt_conf_t   *conf     = NULL;
+
+        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+        GF_VALIDATE_OR_GOTO (this->name, remote_fd, out);
+
+        conf = this->private;
+        pthread_mutex_lock (&conf->lock);
+        {
+                fdctx = this_fd_get_ctx (fd, this);
+                if (!fdctx)
+                        *remote_fd = GF_ANON_FD_NO;
+                else if (__is_fd_reopen_in_progress (fdctx))
+                        *remote_fd = -1;
+                else
+                        *remote_fd = fdctx->remote_fd;
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        if ((flags & FALLBACK_TO_ANON_FD) && (*remote_fd == -1))
+                *remote_fd = GF_ANON_FD_NO;
+
+        return 0;
+out:
+        return -1;
+}
+
+gf_boolean_t
+client_is_reopen_needed (fd_t *fd, xlator_t *this, int64_t remote_fd)
+{
+        clnt_fd_ctx_t   *fdctx = NULL;
+
+        fdctx = this_fd_get_ctx (fd, this);
+        if (fdctx && (fdctx->remote_fd == -1) &&
+            (remote_fd == GF_ANON_FD_NO))
+                return _gf_true;
+        return _gf_false;
+}
+
+int
+client_fd_fop_prepare_local (call_frame_t *frame, fd_t *fd, int64_t remote_fd)
+{
+        xlator_t     *this  = NULL;
+        clnt_conf_t  *conf  = NULL;
+        clnt_local_t *local = NULL;
+        int          ret    = 0;
+
+        this = frame->this;
+        conf = this->private;
+
+        if (!frame || !fd) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        frame->local = mem_get0 (this->local_pool);
+        if (frame->local == NULL) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        local = frame->local;
+        local->fd = fd_ref (fd);
+        local->attempt_reopen = client_is_reopen_needed (fd, this, remote_fd);
+        return 0;
+out:
+        return ret;
+}
diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c
index 4e87f7fcd..1fd8f0d50 100644
--- a/xlators/protocol/client/src/client-lk.c
+++ b/xlators/protocol/client/src/client-lk.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "common-utils.h"
@@ -25,9 +16,6 @@
 static void
 __insert_and_merge (clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock);
 
-static int
-client_send_recovery_lock (call_frame_t *frame, xlator_t *this,
-                           client_posix_lock_t *lock);
 static void
 __dump_client_lock (client_posix_lock_t *lock)
 {
@@ -439,21 +427,6 @@ delete_granted_locks_fd (clnt_fd_ctx_t *fdctx)
         return  ret;
 }
 
-static void
-client_mark_bad_fd (fd_t *fd, clnt_fd_ctx_t *fdctx)
-{
-        xlator_t *this = NULL;
-
-        this = THIS;
-        if (fdctx)
-                fdctx->remote_fd = -1;
-
-        gf_log (this->name, GF_LOG_WARNING,
-                "marking the file descriptor (%p) bad", fd);
-
-        this_fd_set_ctx (fd, this, NULL, fdctx);
-}
-
 int32_t
 client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd)
 {
@@ -562,348 +535,6 @@ out:
 
 }
 
-static int
-construct_reserve_unlock (struct gf_flock *lock, call_frame_t *frame,
-                          client_posix_lock_t *client_lock)
-{
-        GF_ASSERT (lock);
-        GF_ASSERT (frame);
-
-        lock->l_type = F_UNLCK;
-        lock->l_start = 0;
-        lock->l_whence = SEEK_SET;
-        lock->l_len = 0; /* Whole file */
-        lock->l_pid = (uint64_t)(unsigned long)frame->root;
-        lock->l_owner = client_lock->owner;
-
-        frame->root->lk_owner = client_lock->owner;
-
-        return 0;
-}
-
-static int
-construct_reserve_lock (client_posix_lock_t *client_lock, call_frame_t *frame,
-                        struct gf_flock *lock)
-{
-        GF_ASSERT (client_lock);
-
-        memcpy (lock, &(client_lock->user_flock), sizeof (struct gf_flock));
-
-        frame->root->lk_owner = client_lock->owner;
-
-        return 0;
-}
-
-uint64_t
-decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf)
-{
-        uint64_t fd_count = 0;
-
-        LOCK (&conf->rec_lock);
-        {
-                fd_count = --(conf->reopen_fd_count);
-        }
-        UNLOCK (&conf->rec_lock);
-
-        if (fd_count == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "last fd open'd/lock-self-heal'd - notifying CHILD-UP");
-                client_set_lk_version (this);
-                client_notify_parents_child_up (this);
-        }
-
-        return fd_count;
-}
-
-int32_t
-client_remove_reserve_lock_cbk (call_frame_t *frame,
-                                void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret,
-                                int32_t op_errno,
-                                struct gf_flock *lock)
-{
-        clnt_local_t *local = NULL;
-        clnt_conf_t  *conf  = NULL;
-
-        uint64_t fd_count = 0;
-
-        local = frame->local;
-        conf  = this->private;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "removing reserver lock on fd failed: %s",
-                        strerror(op_errno));
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Removing reserve lock was successful.");
-
-cleanup:
-        frame->local = NULL;
-
-        client_mark_bad_fd (local->client_lock->fd, local->fdctx);
-
-        destroy_client_lock (local->client_lock);
-        client_local_wipe (local);
-        STACK_DESTROY (frame->root);
-
-        fd_count = decrement_reopen_fd_count (this, conf);
-        gf_log (this->name, GF_LOG_TRACE,
-                "Need to attempt lock recovery on %lld open fds",
-                (unsigned long long) fd_count);
-	return 0;
-}
-
-static void
-client_remove_reserve_lock (xlator_t *this, call_frame_t *frame,
-                            client_posix_lock_t *lock)
-{
-        struct gf_flock unlock;
-
-        construct_reserve_unlock (&unlock, frame, lock);
-
-        STACK_WIND (frame, client_remove_reserve_lock_cbk,
-                    this, this->fops->lk,
-                    lock->fd, F_RESLK_UNLCK, &unlock);
-}
-
-static client_posix_lock_t *
-get_next_recovery_lock (xlator_t *this, clnt_local_t *local)
-{
-        client_posix_lock_t *lock = NULL;
-
-        pthread_mutex_lock (&local->mutex);
-        {
-                if (list_empty (&local->lock_list)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "lock-list empty");
-                        goto unlock;
-                }
-
-                lock = list_entry ((local->lock_list).next, typeof (*lock), list);
-                list_del_init (&lock->list);
-        }
-unlock:
-        pthread_mutex_unlock (&local->mutex);
-
-        return lock;
-
-}
-
-int32_t
-client_reserve_lock_cbk (call_frame_t *frame,
-                         void *cookie,
-                         xlator_t *this,
-                         int32_t op_ret,
-                         int32_t op_errno,
-                         struct gf_flock *lock)
-{
-
-        clnt_local_t *local = NULL;
-        clnt_conf_t  *conf  = NULL;
-
-        uint64_t fd_count = 0;
-
-        local = frame->local;
-        conf  = this->private;
-
-        /* Got the reserve lock. Check if lock is grantable and proceed
-           with the real lk call */
-
-        if (op_ret >= 0) {
-                /* Lock is grantable if flock reflects a successful getlk() call*/
-                if (lock->l_type == F_UNLCK && lock->l_pid) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Got the reservelk, but the lock is not grantable. ");
-                        client_remove_reserve_lock (this, frame, local->client_lock);
-                        goto out;
-                }
-
-                gf_log (this->name, GF_LOG_DEBUG, "reserve lock succeeded");
-                client_send_recovery_lock (frame, this, local->client_lock);
-                goto out;
-        }
-
-        /* Somebody else has a reserve lk. Lock conflict detected.
-           Mark fd as bad */
-
-        gf_log (this->name, GF_LOG_WARNING,
-                "reservelk OP failed. aborting lock recovery");
-
-        client_mark_bad_fd (local->client_lock->fd,
-                            local->fdctx);
-        destroy_client_lock (local->client_lock);
-        frame->local = NULL;
-        client_local_wipe (local);
-        STACK_DESTROY (frame->root);
-
-        fd_count = decrement_reopen_fd_count (this, conf);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "need to attempt lock recovery on %"PRIu64" open fds",
-                fd_count);
-
-out:
-	return 0;
-}
-
-int32_t
-client_recovery_lock_cbk (call_frame_t *frame,
-                          void *cookie,
-                          xlator_t *this,
-                          int32_t op_ret,
-                          int32_t op_errno,
-                          struct gf_flock *lock)
-{
-        clnt_local_t *local = NULL;
-        clnt_fd_ctx_t *fdctx = NULL;
-        clnt_conf_t   *conf  = NULL;
-        client_posix_lock_t *next_lock = NULL;
-
-        struct gf_flock reserve_flock;
-        uint64_t fd_count = 0;
-
-        local = frame->local;
-        conf  = this->private;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "lock recovery failed: %s",
-                        strerror(op_errno));
-
-                client_mark_bad_fd (local->client_lock->fd,
-                                    local->fdctx);
-                goto cleanup;
-
-                /* Lock recovered. Continue with reserve lock for next lock */
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "lock recovered successfully - continuing with next lock.");
-
-                next_lock = get_next_recovery_lock (this, local);
-                if (!next_lock) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "all locks recovered on fd");
-                        goto cleanup;
-                }
-
-                construct_reserve_lock (next_lock, frame, &reserve_flock);
-                local->fdctx       = fdctx;
-                local->client_lock = next_lock;
-
-                STACK_WIND (frame, client_reserve_lock_cbk,
-                            this, this->fops->lk,
-                            next_lock->fd, F_RESLK_LCK, &reserve_flock);
-                goto out;
-
-        }
-
-cleanup:
-
-        frame->local = NULL;
-        client_local_wipe (local);
-
-        if (local->client_lock)
-                destroy_client_lock (local->client_lock);
-
-        STACK_DESTROY (frame->root);
-
-        fd_count = decrement_reopen_fd_count (this, conf);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "need to attempt lock recovery on %"PRIu64" open fds",
-                fd_count);
-
-out:
-	return 0;
-}
-
-static int
-client_send_recovery_lock (call_frame_t *frame, xlator_t *this,
-                           client_posix_lock_t *lock)
-{
-        frame->root->lk_owner = lock->owner;
-
-        /* Send all locks as F_SETLK to prevent the frame
-           from blocking if there is a conflict */
-
-        STACK_WIND (frame, client_recovery_lock_cbk,
-                    this, this->fops->lk,
-                    lock->fd, F_SETLK,
-                    &(lock->user_flock));
-
-        return 0;
-}
-
-static int
-client_lockrec_init (clnt_fd_ctx_t *fdctx, clnt_local_t *local)
-{
-
-        INIT_LIST_HEAD (&local->lock_list);
-        pthread_mutex_init (&local->mutex, NULL);
-
-        pthread_mutex_lock (&fdctx->mutex);
-        {
-                list_splice_init (&fdctx->lock_list, &local->lock_list);
-        }
-        pthread_mutex_unlock (&fdctx->mutex);
-
-        return 0;
-}
-
-
-int
-client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx)
-{
-        call_frame_t        *frame = NULL;
-        clnt_local_t        *local = NULL;
-        client_posix_lock_t *lock  = NULL;
-
-        struct gf_flock reserve_flock;
-        int ret = 0;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        client_lockrec_init (fdctx, local);
-
-        lock = get_next_recovery_lock (this, local);
-        if (!lock) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no locks found on fd");
-                ret = -1;
-                goto out;
-        }
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "creating of frame failed, lock recovery failed");
-                ret = -1;
-                goto out;
-        }
-
-        construct_reserve_lock (lock, frame, &reserve_flock);
-
-        frame->local       = local;
-        local->fdctx       = fdctx;
-        local->client_lock = lock;
-
-        STACK_WIND (frame, client_reserve_lock_cbk,
-                    this, this->fops->lk,
-                    lock->fd, F_RESLK_LCK, &reserve_flock);
-
-out:
-        return ret;
-
-
-}
-
 int32_t
 client_dump_locks (char *name, inode_t *inode,
                    dict_t *dict)
diff --git a/xlators/protocol/client/src/client-mem-types.h b/xlators/protocol/client/src/client-mem-types.h
index e6e2c8e9f..f6573da2d 100644
--- a/xlators/protocol/client/src/client-mem-types.h
+++ b/xlators/protocol/client/src/client-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client-rpc-fops.c
index b460b167b..99a6f6d74 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client-rpc-fops.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -29,23 +20,25 @@
 
 int32_t client3_getspec (call_frame_t *frame, xlator_t *this, void *data);
 void client_start_ping (void *data);
-rpc_clnt_prog_t clnt3_1_fop_prog;
+rpc_clnt_prog_t clnt3_3_fop_prog;
 
 
 int
 client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
-                           rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbk,
+                           rpc_clnt_prog_t *prog, int procnum,
+                           fop_cbk_fn_t cbkfn,
                            struct iovec  *payload, int payloadcnt,
                            struct iobref *iobref, xdrproc_t xdrproc)
 {
-        int            ret        = 0;
-        clnt_conf_t   *conf       = NULL;
-        struct iovec   iov        = {0, };
-        struct iobuf  *iobuf      = NULL;
-        int            count      = 0;
-        int            start_ping = 0;
-        struct iobref *new_iobref = NULL;
-        ssize_t        xdr_size   = 0;
+        int             ret        = 0;
+        clnt_conf_t    *conf       = NULL;
+        struct iovec    iov        = {0, };
+        struct iobuf   *iobuf      = NULL;
+        int             count      = 0;
+        int             start_ping = 0;
+        struct iobref  *new_iobref = NULL;
+        ssize_t         xdr_size   = 0;
+        struct rpc_req  rpcreq     = {0, };
 
         start_ping = 0;
 
@@ -55,12 +48,12 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
                 xdr_size = xdr_sizeof (xdrproc, req);
                 iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
                 if (!iobuf) {
-                        goto out;
+                        goto unwind;
                 };
 
                 new_iobref = iobref_new ();
                 if (!new_iobref) {
-                        goto out;
+                        goto unwind;
                 }
 
                 if (iobref != NULL) {
@@ -76,7 +69,7 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
                 if (ret != 0) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "cannot add iobuf into iobref");
-                        goto out;
+                        goto unwind;
                 }
 
                 iov.iov_base = iobuf->ptr;
@@ -87,7 +80,7 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
                 if (ret == -1) {
                         gf_log_callingfn ("", GF_LOG_WARNING,
                                           "XDR function failed");
-                        goto out;
+                        goto unwind;
                 }
 
                 iov.iov_len = ret;
@@ -95,7 +88,7 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
         }
 
         /* Send the msg */
-        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count,
+        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbkfn, &iov, count,
                                payload, payloadcnt, new_iobref, frame, NULL, 0,
                                NULL, 0, NULL);
         if (ret < 0) {
@@ -115,12 +108,23 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
         if (start_ping)
                 client_start_ping ((void *) this);
 
-out:
-        if (new_iobref != NULL) {
+        if (new_iobref)
                 iobref_unref (new_iobref);
-        }
 
-        iobuf_unref (iobuf);
+        if (iobuf)
+                iobuf_unref (iobuf);
+
+        return ret;
+
+unwind:
+        rpcreq.rpc_status = -1;
+        cbkfn (&rpcreq, NULL, 0, frame);
+
+        if (new_iobref)
+                iobref_unref (new_iobref);
+
+        if (iobuf)
+                iobuf_unref (iobuf);
 
         return ret;
 }
@@ -128,7 +132,7 @@ out:
 /* CBK */
 
 int
-client3_1_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -140,13 +144,13 @@ client3_1_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
         clnt_local_t     *local      = NULL;
         inode_t          *inode      = NULL;
         xlator_t         *this       = NULL;
+        dict_t           *xdata      = NULL;
 
         this = THIS;
 
         frame = myframe;
 
         local = frame->local;
-        frame->local = NULL;
         inode = local->loc.inode;
 
         if (-1 == req->rpc_status) {
@@ -169,26 +173,36 @@ client3_1_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
+                /* no need to print the gfid, because it will be null, since
+                 * symlink operation failed.
+                 */
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s. Path: (%s to %s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, local->loc2.path);
         }
-        STACK_UNWIND_STRICT (symlink, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (symlink, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), inode, &stbuf,
-                             &preparent, &postparent);
+                             &preparent, &postparent, xdata);
 
-         client_local_wipe (local);
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -200,13 +214,14 @@ client3_1_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
         clnt_local_t     *local      = NULL;
         inode_t          *inode      = NULL;
         xlator_t         *this       = NULL;
+        dict_t           *xdata      = NULL;
 
         this = THIS;
 
         frame = myframe;
 
         local = frame->local;
-        frame->local = NULL;
+
         inode = local->loc.inode;
 
         if (-1 == req->rpc_status) {
@@ -229,25 +244,32 @@ client3_1_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s. Path: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path);
         }
-        STACK_UNWIND_STRICT (mknod, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (mknod, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent);
+                             &stbuf, &preparent, &postparent, xdata);
 
-        client_local_wipe (local);
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -259,6 +281,7 @@ client3_1_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         clnt_local_t     *local      = NULL;
         inode_t          *inode      = NULL;
         xlator_t         *this       = NULL;
+        dict_t           *xdata      = NULL;
 
         this = THIS;
 
@@ -287,42 +310,122 @@ client3_1_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s. Path: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path);
         }
-        STACK_UNWIND_STRICT (mkdir, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (mkdir, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent);
+                             &stbuf, &preparent, &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
 
-        client_local_wipe (local);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
+_copy_gfid_from_inode_holders (uuid_t gfid, loc_t *loc, fd_t *fd)
+{
+        int     ret = 0;
+
+        if (fd && fd->inode && !uuid_is_null (fd->inode->gfid)) {
+                uuid_copy (gfid, fd->inode->gfid);
+                goto out;
+        }
+
+        if (!loc) {
+                GF_ASSERT (0);
+                ret = -1;
+                goto out;
+        }
+
+        if (loc->inode && !uuid_is_null (loc->inode->gfid)) {
+                uuid_copy (gfid, loc->inode->gfid);
+        } else if (!uuid_is_null (loc->gfid)) {
+                uuid_copy (gfid, loc->gfid);
+        } else {
+                GF_ASSERT (0);
+                ret = -1;
+        }
+out:
+        return ret;
+}
+
+int
+client_add_fd_to_saved_fds (xlator_t *this, fd_t *fd, loc_t *loc, int32_t flags,
+                            int64_t remote_fd, int is_dir)
+{
+        int             ret = 0;
+        uuid_t          gfid = {0};
+        clnt_conf_t     *conf       = NULL;
+        clnt_fd_ctx_t   *fdctx      = NULL;
+
+        conf  = this->private;
+        ret = _copy_gfid_from_inode_holders (gfid, loc, fd);
+        if (ret) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        fdctx = GF_CALLOC (1, sizeof (*fdctx),
+                           gf_client_mt_clnt_fdctx_t);
+        if (!fdctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        uuid_copy (fdctx->gfid, gfid);
+        fdctx->is_dir        = is_dir;
+        fdctx->remote_fd     = remote_fd;
+        fdctx->flags         = flags;
+        fdctx->lk_ctx        = fd_lk_ctx_ref (fd->lk_ctx);
+        fdctx->lk_heal_state = GF_LK_HEAL_DONE;
+        fdctx->reopen_done   = client_default_reopen_done;
+
+        INIT_LIST_HEAD (&fdctx->sfd_pos);
+        INIT_LIST_HEAD (&fdctx->lock_list);
+
+        this_fd_set_ctx (fd, this, loc, fdctx);
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
+        }
+        pthread_mutex_unlock (&conf->lock);
+out:
+        return ret;
+}
+
+int
+client3_3_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
                     void *myframe)
 {
         clnt_local_t  *local = NULL;
         clnt_conf_t   *conf  = NULL;
-        clnt_fd_ctx_t *fdctx = NULL;
         call_frame_t  *frame = NULL;
         fd_t          *fd    = NULL;
         int            ret   = 0;
         gfs3_open_rsp  rsp   = {0,};
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
 
-        frame->local = NULL;
         conf  = frame->this->private;
         fd    = local->fd;
 
@@ -340,58 +443,50 @@ client3_1_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
         if (-1 != rsp.op_ret) {
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_clnt_fdctx_t);
-                if (!fdctx) {
+                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
+                                                  local->flags, rsp.fd, 0);
+                if (ret) {
                         rsp.op_ret = -1;
-                        rsp.op_errno = ENOMEM;
+                        rsp.op_errno = -ret;
                         goto out;
                 }
-
-                fdctx->remote_fd = rsp.fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->flags     = local->flags;
-                fdctx->wbflags   = local->wbflags;
-                fdctx->lk_ctx = fd_lk_ctx_ref (fd->lk_ctx);
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-                INIT_LIST_HEAD (&fdctx->lock_list);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->lock);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->lock);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s. Path: %s (%s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, loc_gfid_utoa (&local->loc));
         }
-        STACK_UNWIND_STRICT (open, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), fd);
 
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (open, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), fd, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_stat_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_stat_cbk (struct rpc_req *req, struct iovec *iov, int count,
                     void *myframe)
 {
         gfs3_stat_rsp rsp = {0,};
         call_frame_t   *frame = NULL;
         struct iatt  iatt = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -414,26 +509,38 @@ client3_1_stat_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.stat, &iatt);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (stat, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &iatt);
+
+        CLIENT_STACK_UNWIND (stat, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &iatt, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_readlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         gfs3_readlink_rsp rsp = {0,};
         call_frame_t   *frame = NULL;
         struct iatt  iatt = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -456,24 +563,35 @@ client3_1_readlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.buf, &iatt);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+                gf_log (this->name, (gf_error_to_errno(rsp.op_errno) == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_WARNING, "remote operation failed:"
+                        " %s", strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (readlink, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), rsp.path, &iatt);
+
+        CLIENT_STACK_UNWIND (readlink, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), rsp.path,
+                             &iatt, xdata);
 
         /* This is allocated by the libc while decoding RPC msg */
         /* Hence no 'GF_FREE', but just 'free' */
-        if (rsp.path)
-                free (rsp.path);
+        free (rsp.path);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         call_frame_t    *frame      = NULL;
@@ -481,7 +599,9 @@ client3_1_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt      preparent  = {0,};
         struct iatt      postparent = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -505,20 +625,33 @@ client3_1_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
+                gf_log (this->name,
+                        ((gf_error_to_errno (rsp.op_errno) == ENOENT)
+                        ? GF_LOG_DEBUG : GF_LOG_WARNING),
+                        "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (unlink, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (unlink, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &preparent,
-                             &postparent);
+                             &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         gfs3_rmdir_rsp rsp = {0,};
@@ -526,7 +659,9 @@ client3_1_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  preparent  = {0,};
         struct iatt  postparent = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -550,21 +685,30 @@ client3_1_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (rmdir, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (rmdir, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &preparent,
-                             &postparent);
+                             &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         gfs3_truncate_rsp rsp = {0,};
@@ -572,7 +716,9 @@ client3_1_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  prestat  = {0,};
         struct iatt  poststat = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -596,28 +742,39 @@ client3_1_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.poststat, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (truncate, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (truncate, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_statfs_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_statfs_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         gfs3_statfs_rsp rsp = {0,};
         call_frame_t   *frame = NULL;
         struct statvfs  statfs = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -640,20 +797,29 @@ client3_1_statfs_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_statfs_to_statfs (&rsp.statfs, &statfs);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (statfs, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &statfs);
+        CLIENT_STACK_UNWIND (statfs, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &statfs, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         gfs3_write_rsp rsp = {0,};
@@ -661,11 +827,15 @@ client3_1_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  prestat  = {0,};
         struct iatt  poststat = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+        clnt_local_t    *local = NULL;
+
 
         this = THIS;
 
         frame = myframe;
+        local = frame->local;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -686,25 +856,39 @@ client3_1_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.poststat, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret >= 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
-        STACK_UNWIND_STRICT (writev, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (writev, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_flush_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_flush_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         call_frame_t    *frame      = NULL;
         clnt_local_t  *local      = NULL;
-        xlator_t        *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
 
@@ -733,23 +917,29 @@ client3_1_flush_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         lkowner_utoa (&local->owner), ret);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (flush, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (flush, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
 
-        client_local_wipe (local);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         gfs3_fsync_rsp rsp = {0,};
@@ -757,7 +947,9 @@ client3_1_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  prestat  = {0,};
         struct iatt  poststat = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -782,26 +974,37 @@ client3_1_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.poststat, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fsync, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (fsync, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_setxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_setxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t         *this       = NULL;
+        call_frame_t  *frame    = NULL;
+        gf_common_rsp  rsp      = {0,};
+        int            ret      = 0;
+        xlator_t      *this     = NULL;
+        dict_t        *xdata    = NULL;
+        int            op_errno = EINVAL;
 
         this = THIS;
 
@@ -821,19 +1024,30 @@ client3_1_setxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
+        op_errno = gf_error_to_errno (rsp.op_errno);
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+                gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_WARNING),
+                        "remote operation failed: %s",
+                        strerror (op_errno));
         }
-        STACK_UNWIND_STRICT (setxattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (setxattr, frame, rsp.op_ret, op_errno, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t      *frame    = NULL;
@@ -842,13 +1056,14 @@ client3_1_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
         gfs3_getxattr_rsp  rsp      = {0,};
         int                ret      = 0;
         clnt_local_t    *local    = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret = -1;
@@ -872,32 +1087,40 @@ client3_1_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                                               op_errno, out);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                gf_log (this->name, (((op_errno == ENOTSUP) ||
+                                      (op_errno == ENODATA) ||
+                                      (op_errno == ENOENT)) ?
+                                     GF_LOG_DEBUG : GF_LOG_WARNING),
+                        "remote operation failed: %s. Path: %s (%s). Key: %s",
                         strerror (op_errno),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, loc_gfid_utoa (&local->loc),
+                        (local->name) ? local->name : "(null)");
         }
-        STACK_UNWIND_STRICT (getxattr, frame, rsp.op_ret, op_errno, dict);
 
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-                rsp.dict.dict_val = NULL;
-        }
+        CLIENT_STACK_UNWIND (getxattr, frame, rsp.op_ret, op_errno, dict, xdata);
+
+        /* don't use GF_FREE, this memory was allocated by libc */
+        free (rsp.dict.dict_val);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         if (dict)
                 dict_unref (dict);
 
-        client_local_wipe (local);
-
         return 0;
 }
 
 int
-client3_1_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void *myframe)
 {
         call_frame_t       *frame    = NULL;
@@ -906,13 +1129,14 @@ client3_1_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
         int                 ret      = 0;
         int                 op_errno = EINVAL;
         clnt_local_t     *local    = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret = -1;
@@ -934,36 +1158,42 @@ client3_1_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                                               (rsp.dict.dict_len), rsp.op_ret,
                                               op_errno, out);
         }
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_WARNING),
                         "remote operation failed: %s",
                         strerror (op_errno));
         }
-        STACK_UNWIND_STRICT (fgetxattr, frame, rsp.op_ret, op_errno, dict);
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-                rsp.dict.dict_val = NULL;
-        }
+
+        CLIENT_STACK_UNWIND (fgetxattr, frame, rsp.op_ret, op_errno, dict, xdata);
+
+        free (rsp.dict.dict_val);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         if (dict)
                 dict_unref (dict);
 
-        client_local_wipe (local);
-
         return 0;
 }
 
 int
-client3_1_removexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_removexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                            void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
@@ -983,25 +1213,37 @@ client3_1_removexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (removexattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+
+        CLIENT_STACK_UNWIND (removexattr, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fremovexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fremovexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                             void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1021,25 +1263,36 @@ client3_1_fremovexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fremovexattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (fremovexattr, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fsyncdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fsyncdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1058,25 +1311,36 @@ client3_1_fsyncdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fsyncdir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (fsyncdir, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_access_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_access_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1095,20 +1359,29 @@ client3_1_access_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (access, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (access, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void *myframe)
 {
         gfs3_ftruncate_rsp rsp = {0,};
@@ -1116,7 +1389,9 @@ client3_1_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt  prestat  = {0,};
         struct iatt  poststat = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1140,27 +1415,38 @@ client3_1_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.poststat, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (ftruncate, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (ftruncate, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fstat_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fstat_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         gfs3_fstat_rsp rsp = {0,};
         call_frame_t   *frame = NULL;
         struct iatt  stat  = {0,};
         int ret = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1183,26 +1469,37 @@ client3_1_fstat_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.stat, &stat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fstat, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &stat);
+        CLIENT_STACK_UNWIND (fstat, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &stat,  xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_inodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_inodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1221,30 +1518,42 @@ client3_1_inodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (inodelk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (inodelk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_finodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_finodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t         *this       = NULL;
+        call_frame_t  *frame = NULL;
+        gf_common_rsp rsp    = {0,};
+        int           ret    = 0;
+        xlator_t      *this  = NULL;
+        dict_t        *xdata = NULL;
+        clnt_local_t  *local = NULL;
 
-        this = THIS;
 
         frame = myframe;
+        this = frame->this;
+        local = frame->local;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1259,26 +1568,40 @@ client3_1_finodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret == 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
-        STACK_UNWIND_STRICT (finodelk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (finodelk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_entrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_entrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1297,27 +1620,38 @@ client3_1_entrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
-out:
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
 
+out:
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (entrylk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+
+        CLIENT_STACK_UNWIND (entrylk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fentrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fentrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t    *frame      = NULL;
         gf_common_rsp    rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1336,20 +1670,30 @@ client3_1_fentrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fentrylk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+
+        CLIENT_STACK_UNWIND (fentrylk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t     *frame    = NULL;
@@ -1358,13 +1702,14 @@ client3_1_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
         int               ret      = 0;
         int               op_errno = EINVAL;
         clnt_local_t   *local    = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret = -1;
@@ -1387,49 +1732,51 @@ client3_1_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
                                               op_errno, out);
         }
 
-out:
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      op_errno, out);
 
+out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s. Path: %s (%s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, loc_gfid_utoa (&local->loc));
         }
-        STACK_UNWIND_STRICT (xattrop, frame, rsp.op_ret,
-                             gf_error_to_errno (op_errno), dict);
 
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-                rsp.dict.dict_val = NULL;
-        }
+        CLIENT_STACK_UNWIND (xattrop, frame, rsp.op_ret,
+                             gf_error_to_errno (op_errno), dict, xdata);
+
+        free (rsp.dict.dict_val);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         if (dict)
                 dict_unref (dict);
 
-        client_local_wipe (local);
-
         return 0;
 }
 
 int
-client3_1_fxattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fxattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t      *frame    = NULL;
         dict_t            *dict     = NULL;
+        dict_t            *xdata    = NULL;
         gfs3_fxattrop_rsp  rsp      = {0,};
         int                ret      = 0;
         int                op_errno = 0;
         clnt_local_t    *local    = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1452,38 +1799,99 @@ client3_1_fxattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
                                               op_errno, out);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
+                                      (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), rsp.op_ret,
+                                      op_errno, out);
 out:
 
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (op_errno)));
+        } else if (rsp.op_ret == 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
-        STACK_UNWIND_STRICT (fxattrop, frame, rsp.op_ret,
-                             gf_error_to_errno (op_errno), dict);
+        CLIENT_STACK_UNWIND (fxattrop, frame, rsp.op_ret,
+                             gf_error_to_errno (op_errno), dict, xdata);
 
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-                rsp.dict.dict_val = NULL;
-        }
+        free (rsp.dict.dict_val);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         if (dict)
                 dict_unref (dict);
 
-        client_local_wipe (local);
         return 0;
 }
 
 int
-client3_1_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void *myframe)
 {
+        call_frame_t  *frame    = NULL;
+        gf_common_rsp  rsp      = {0,};
+        int            ret      = 0;
+        xlator_t      *this     = NULL;
+        dict_t        *xdata    = NULL;
+        int            op_errno = EINVAL;
+
+        this = THIS;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+out:
+        op_errno = gf_error_to_errno (rsp.op_errno);
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_WARNING),
+                        "remote operation failed: %s",
+                        strerror (op_errno));
+        }
+
+        CLIENT_STACK_UNWIND (fsetxattr, frame, rsp.op_ret, op_errno, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
+
+int
+client3_3_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
         call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
+        gfs3_fsetattr_rsp rsp        = {0,};
+        struct iatt      prestat    = {0,};
+        struct iatt      poststat   = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1494,7 +1902,7 @@ client3_1_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 rsp.op_errno = ENOTCONN;
                 goto out;
         }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fsetattr_rsp);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
                 rsp.op_ret   = -1;
@@ -1502,27 +1910,44 @@ client3_1_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        if (-1 != rsp.op_ret) {
+                gf_stat_to_iatt (&rsp.statpre, &prestat);
+                gf_stat_to_iatt (&rsp.statpost, &poststat);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fsetxattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno));
+        CLIENT_STACK_UNWIND (fsetattr, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &prestat,
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fallocate_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t    *frame      = NULL;
-        gfs3_fsetattr_rsp rsp        = {0,};
+        gfs3_fallocate_rsp rsp      = {0,};
         struct iatt      prestat    = {0,};
         struct iatt      poststat   = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1533,7 +1958,7 @@ client3_1_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 rsp.op_errno = ENOTCONN;
                 goto out;
         }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fsetattr_rsp);
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fallocate_rsp);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
                 rsp.op_ret   = -1;
@@ -1546,21 +1971,140 @@ client3_1_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.statpost, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (fsetattr, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (fallocate, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
+int
+client3_3_discard_cbk(struct rpc_req *req, struct iovec *iov, int count,
+		      void *myframe)
+{
+        call_frame_t    *frame      = NULL;
+        gfs3_discard_rsp rsp      = {0,};
+        struct iatt      prestat    = {0,};
+        struct iatt      poststat   = {0,};
+        int              ret        = 0;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
+        this = THIS;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_discard_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        if (-1 != rsp.op_ret) {
+                gf_stat_to_iatt (&rsp.statpre, &prestat);
+                gf_stat_to_iatt (&rsp.statpost, &poststat);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+out:
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
+                        strerror (gf_error_to_errno (rsp.op_errno)));
+        }
+        CLIENT_STACK_UNWIND (discard, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &prestat,
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
 
 int
-client3_1_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_zerofill_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+        call_frame_t    *frame         = NULL;
+        gfs3_zerofill_rsp rsp          = {0,};
+        struct iatt      prestat       = {0,};
+        struct iatt      poststat      = {0,};
+        int              ret           = 0;
+        xlator_t *this                 = NULL;
+        dict_t  *xdata                 = NULL;
+
+        this = THIS;
+
+        frame = myframe;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = ENOTCONN;
+                goto out;
+        }
+        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_zerofill_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        if (-1 != rsp.op_ret) {
+                gf_stat_to_iatt (&rsp.statpre, &prestat);
+                gf_stat_to_iatt (&rsp.statpost, &poststat);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+out:
+        if (rsp.op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "remote operation failed: %s",
+                        strerror (gf_error_to_errno (rsp.op_errno)));
+        }
+        CLIENT_STACK_UNWIND (zerofill, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &prestat,
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
+
+int
+client3_3_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t    *frame      = NULL;
@@ -1568,7 +2112,9 @@ client3_1_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt      prestat    = {0,};
         struct iatt      poststat   = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1593,20 +2139,29 @@ client3_1_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.statpost, &poststat);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (setattr, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (setattr, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat);
+                             &poststat, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         call_frame_t    *frame      = NULL;
@@ -1617,16 +2172,14 @@ client3_1_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt      postparent = {0, };
         int32_t          ret        = -1;
         clnt_local_t    *local      = NULL;
-        clnt_conf_t     *conf       = NULL;
-        clnt_fd_ctx_t   *fdctx      = NULL;
         gfs3_create_rsp  rsp        = {0,};
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
         frame = myframe;
-        local = frame->local; frame->local = NULL;
-        conf  = frame->this->private;
+        local = frame->local;
         fd    = local->fd;
         inode = local->loc.inode;
 
@@ -1649,56 +2202,51 @@ client3_1_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
 
                 gf_stat_to_iatt (&rsp.preparent, &preparent);
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
-
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_clnt_fdctx_t);
-                if (!fdctx) {
+                uuid_copy (local->loc.gfid, stbuf.ia_gfid);
+                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
+                                                  local->flags, rsp.fd, 0);
+                if (ret) {
                         rsp.op_ret = -1;
-                        rsp.op_errno = ENOMEM;
+                        rsp.op_errno = -ret;
                         goto out;
                 }
-
-                fdctx->remote_fd = rsp.fd;
-                fdctx->inode     = inode_ref (inode);
-                fdctx->flags     = local->flags;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-                INIT_LIST_HEAD (&fdctx->lock_list);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->lock);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->lock);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s. Path: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path);
         }
-        STACK_UNWIND_STRICT (create, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (create, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), fd, inode,
-                             &stbuf, &preparent, &postparent);
+                             &stbuf, &preparent, &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
-        client_local_wipe (local);
         return 0;
 }
 
 
 int
-client3_1_rchecksum_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_rchecksum_cbk (struct rpc_req *req, struct iovec *iov, int count,
                          void *myframe)
 {
         call_frame_t *frame = NULL;
         gfs3_rchecksum_rsp rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
@@ -1718,15 +2266,20 @@ client3_1_rchecksum_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 goto out;
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (rchecksum, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (rchecksum, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno),
                              rsp.weak_checksum,
-                             (uint8_t *)rsp.strong_checksum.strong_checksum_val);
+                             (uint8_t *)rsp.strong_checksum.strong_checksum_val,
+                             xdata);
 
         if (rsp.strong_checksum.strong_checksum_val) {
                 /* This is allocated by the libc while decoding RPC msg */
@@ -1734,11 +2287,16 @@ out:
                 free (rsp.strong_checksum.strong_checksum_val);
         }
 
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
         return 0;
 }
 
 int
-client3_1_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
                   void *myframe)
 {
         call_frame_t    *frame      = NULL;
@@ -1746,7 +2304,8 @@ client3_1_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct gf_flock     lock       = {0,};
         gfs3_lk_rsp      rsp        = {0,};
         int              ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
@@ -1785,37 +2344,47 @@ client3_1_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
         */
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if ((rsp.op_ret == -1) &&
             (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (lk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &lock);
 
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (lk, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &lock, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        free (rsp.flock.lk_owner.lk_owner_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
         return 0;
 }
 
 int
-client3_1_readdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
-        call_frame_t           *frame = NULL;
-        gfs3_readdir_rsp        rsp   = {0,};
-        int32_t                 ret   = 0;
-        clnt_local_t         *local = NULL;
-        gf_dirent_t             entries;
-        xlator_t         *this       = NULL;
+        call_frame_t     *frame    = NULL;
+        gfs3_readdir_rsp  rsp      = {0,};
+        int32_t           ret      = 0;
+        clnt_local_t     *local    = NULL;
+        gf_dirent_t       entries;
+        xlator_t         *this     = NULL;
+        dict_t           *xdata    = NULL;
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1836,6 +2405,11 @@ client3_1_readdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 unserialize_rsp_dirent (&rsp, &entries);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
+                                      (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), rsp.op_ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
@@ -1843,15 +2417,18 @@ out:
                         strerror (gf_error_to_errno (rsp.op_errno)),
                                   local->cmd);
         }
-        STACK_UNWIND_STRICT (readdir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &entries);
-
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (readdir, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &entries, xdata);
 
         if (rsp.op_ret != -1) {
                 gf_dirent_free (&entries);
         }
 
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
+
         clnt_readdir_rsp_cleanup (&rsp);
 
         return 0;
@@ -1859,7 +2436,7 @@ out:
 
 
 int
-client3_1_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
         call_frame_t      *frame = NULL;
@@ -1868,12 +2445,12 @@ client3_1_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
         clnt_local_t      *local = NULL;
         gf_dirent_t        entries;
         xlator_t          *this  = NULL;
+        dict_t            *xdata = NULL;
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -1894,20 +2471,26 @@ client3_1_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 unserialize_rsp_direntp (this, local->fd, &rsp, &entries);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (readdirp, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &entries);
-
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (readdirp, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), &entries, xdata);
 
         if (rsp.op_ret != -1) {
                 gf_dirent_free (&entries);
         }
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         clnt_readdirp_rsp_cleanup (&rsp);
 
@@ -1916,7 +2499,7 @@ out:
 
 
 int
-client3_1_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -1927,7 +2510,8 @@ client3_1_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt       prenewparent  = {0,};
         struct iatt       postnewparent = {0,};
         int               ret        = 0;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
@@ -1957,21 +2541,30 @@ client3_1_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postnewparent, &postnewparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
         }
-        STACK_UNWIND_STRICT (rename, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (rename, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno),
                              &stbuf, &preoldparent, &postoldparent,
-                             &prenewparent, &postnewparent);
+                             &prenewparent, &postnewparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
                     void *myframe)
 {
         call_frame_t     *frame      = NULL;
@@ -1982,14 +2575,15 @@ client3_1_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
         int               ret        = 0;
         clnt_local_t     *local      = NULL;
         inode_t          *inode      = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
+
 
         this = THIS;
 
         frame = myframe;
 
         local = frame->local;
-        frame->local = NULL;
         inode = local->loc.inode;
 
         if (-1 == req->rpc_status) {
@@ -2013,42 +2607,49 @@ client3_1_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
                 gf_stat_to_iatt (&rsp.postparent, &postparent);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s (%s -> %s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, local->loc2.path);
         }
-        STACK_UNWIND_STRICT (link, frame, rsp.op_ret,
+
+        CLIENT_STACK_UNWIND (link, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent);
+                             &stbuf, &preparent, &postparent, xdata);
+
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
-        client_local_wipe (local);
         return 0;
 }
 
 
 int
-client3_1_opendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_opendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         clnt_local_t      *local = NULL;
         clnt_conf_t       *conf = NULL;
-        clnt_fd_ctx_t     *fdctx = NULL;
-        call_frame_t   *frame = NULL;
-        fd_t             *fd = NULL;
+        call_frame_t      *frame = NULL;
+        fd_t              *fd = NULL;
         int ret = 0;
         gfs3_opendir_rsp  rsp = {0,};
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
         frame = myframe;
         local = frame->local;
 
-        frame->local = NULL;
         conf  = frame->this->private;
         fd    = local->fd;
 
@@ -2067,49 +2668,40 @@ client3_1_opendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
         if (-1 != rsp.op_ret) {
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_clnt_fdctx_t);
-                if (!fdctx) {
+                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
+                                                  0, rsp.fd, 1);
+                if (ret) {
                         rsp.op_ret = -1;
-                        rsp.op_errno = ENOMEM;
+                        rsp.op_errno = -ret;
                         goto out;
                 }
-
-                fdctx->remote_fd = rsp.fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->is_dir    = 1;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-                INIT_LIST_HEAD (&fdctx->lock_list);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->lock);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->lock);
         }
 
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
 out:
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
+                        "remote operation failed: %s. Path: %s (%s)",
                         strerror (gf_error_to_errno (rsp.op_errno)),
-                        (local) ? local->loc.path : "--");
+                        local->loc.path, loc_gfid_utoa (&local->loc));
         }
-        STACK_UNWIND_STRICT (opendir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), fd);
+        CLIENT_STACK_UNWIND (opendir, frame, rsp.op_ret,
+                             gf_error_to_errno (rsp.op_errno), fd, xdata);
 
-        client_local_wipe (local);
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 
 int
-client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
         clnt_local_t    *local      = NULL;
@@ -2119,7 +2711,7 @@ client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
         struct iatt      stbuf      = {0,};
         struct iatt      postparent = {0,};
         int              op_errno   = EINVAL;
-        dict_t          *xattr      = NULL;
+        dict_t          *xdata      = NULL;
         inode_t         *inode      = NULL;
         xlator_t        *this       = NULL;
 
@@ -2128,7 +2720,6 @@ client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
         frame = myframe;
         local = frame->local;
         inode = local->loc.inode;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -2153,8 +2744,8 @@ client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
         rsp.op_ret = -1;
         gf_stat_to_iatt (&rsp.stat, &stbuf);
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xattr, (rsp.dict.dict_val),
-                                      (rsp.dict.dict_len), rsp.op_ret,
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), rsp.op_ret,
                                       op_errno, out);
 
         if ((!uuid_is_null (inode->gfid))
@@ -2170,37 +2761,32 @@ client3_1_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
 
 out:
         rsp.op_errno = op_errno;
-        frame->local = NULL;
         if (rsp.op_ret == -1) {
                 /* any error other than ENOENT */
-                if (rsp.op_errno != ENOENT)
+                if (!(local->loc.name && rsp.op_errno == ENOENT) &&
+		    !(rsp.op_errno == ESTALE))
                         gf_log (this->name, GF_LOG_WARNING,
-                                "remote operation failed: %s. Path: %s",
-                                strerror (rsp.op_errno),
-                                (local) ? local->loc.path : "--");
+                                "remote operation failed: %s. Path: %s (%s)",
+                                strerror (rsp.op_errno), local->loc.path,
+                                loc_gfid_utoa (&local->loc));
                 else
                         gf_log (this->name, GF_LOG_TRACE, "not found on remote node");
 
         }
-        STACK_UNWIND_STRICT (lookup, frame, rsp.op_ret, rsp.op_errno, inode,
-                             &stbuf, xattr, &postparent);
 
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (lookup, frame, rsp.op_ret, rsp.op_errno, inode,
+                             &stbuf, xdata, &postparent);
 
-        if (xattr)
-                dict_unref (xattr);
+        if (xdata)
+                dict_unref (xdata);
 
-        if (rsp.dict.dict_val) {
-                /* don't use GF_FREE, this memory was allocated by libc
-                 */
-                free (rsp.dict.dict_val);
-        }
+        free (rsp.xdata.xdata_val);
 
         return 0;
 }
 
 int
-client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
         call_frame_t   *frame  = NULL;
@@ -2210,7 +2796,8 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
         gfs3_read_rsp   rsp    = {0,};
         int             ret    = 0, rspcount = 0;
         clnt_local_t   *local  = NULL;
-        xlator_t         *this       = NULL;
+        xlator_t *this       = NULL;
+        dict_t  *xdata       = NULL;
 
         this = THIS;
 
@@ -2218,7 +2805,6 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
 
         frame = myframe;
         local = frame->local;
-        frame->local = NULL;
 
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
@@ -2243,23 +2829,37 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         vector[0].iov_base = req->rsp[1].iov_base;
                 rspcount = 1;
         }
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
+                                      (rsp.xdata.xdata_len), ret,
+                                      rsp.op_errno, out);
+
+#ifdef GF_TESTING_IO_XDATA
+        dict_dump (xdata);
+#endif
+
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "remote operation failed: %s",
                         strerror (gf_error_to_errno (rsp.op_errno)));
+        } else if (rsp.op_ret >= 0) {
+                if (local->attempt_reopen)
+                        client_attempt_reopen (local->fd, this);
         }
-        STACK_UNWIND_STRICT (readv, frame, rsp.op_ret,
+        CLIENT_STACK_UNWIND (readv, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), vector, rspcount,
-                             &stat, iobref);
+                             &stat, iobref, xdata);
 
-        client_local_wipe (local);
+        free (rsp.xdata.xdata_val);
+
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
 int
-client3_1_release_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_release_cbk (struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
         call_frame_t   *frame = NULL;
@@ -2269,7 +2869,7 @@ client3_1_release_cbk (struct rpc_req *req, struct iovec *iov, int count,
         return 0;
 }
 int
-client3_1_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
                           void *myframe)
 {
         call_frame_t   *frame = NULL;
@@ -2282,13 +2882,14 @@ client3_1_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
 int
 client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx)
 {
-        clnt_conf_t  *conf = NULL;
-        call_frame_t *fr = NULL;
-        int32_t       ret = -1;
-        fd_lk_ctx_t  *lk_ctx = NULL;
+        clnt_conf_t  *conf        = NULL;
+        call_frame_t *fr          = NULL;
+        int32_t       ret         = -1;
+        char          parent_down = 0;
+        fd_lk_ctx_t  *lk_ctx      = NULL;
 
-        if (!fdctx)
-                goto out;
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
 
         conf = (clnt_conf_t *) this->private;
 
@@ -2299,39 +2900,51 @@ client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx)
 
         pthread_mutex_lock (&conf->lock);
         {
-                lk_ctx = fdctx->lk_ctx;
+                parent_down   = conf->parent_down;
+                lk_ctx        = fdctx->lk_ctx;
                 fdctx->lk_ctx = NULL;
         }
         pthread_mutex_unlock (&conf->lock);
 
-        fd_lk_ctx_unref (lk_ctx);
+        if (lk_ctx)
+                fd_lk_ctx_unref (lk_ctx);
+
+        if (!parent_down)
+                rpc_clnt_ref (conf->rpc);
+        else
+                goto out;
 
         fr = create_frame (this, this->ctx->pool);
+        if (fr == NULL) {
+                goto out;
+        }
+
+        ret = 0;
 
         if (fdctx->is_dir) {
                 gfs3_releasedir_req  req = {{0,},};
                 req.fd = fdctx->remote_fd;
-                gf_log (this->name, GF_LOG_INFO, "sending releasedir on fd");
-                ret = client_submit_request (this, &req, fr, &clnt3_1_fop_prog,
-                                             GFS3_OP_RELEASEDIR,
-                                             client3_1_releasedir_cbk,
-                                             NULL, NULL, 0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_releasedir_req);
+                gf_log (this->name, GF_LOG_TRACE, "sending releasedir on fd");
+                client_submit_request (this, &req, fr, &clnt3_3_fop_prog,
+                                       GFS3_OP_RELEASEDIR,
+                                       client3_3_releasedir_cbk,
+                                       NULL, NULL, 0, NULL, 0, NULL,
+                                       (xdrproc_t)xdr_gfs3_releasedir_req);
         } else {
                 gfs3_release_req  req = {{0,},};
                 req.fd = fdctx->remote_fd;
-                gf_log (this->name, GF_LOG_INFO, "sending release on fd");
-                ret = client_submit_request (this, &req, fr, &clnt3_1_fop_prog,
-                                             GFS3_OP_RELEASE,
-                                             client3_1_release_cbk, NULL,
-                                             NULL, 0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_release_req);
+                gf_log (this->name, GF_LOG_TRACE, "sending release on fd");
+                client_submit_request (this, &req, fr, &clnt3_3_fop_prog,
+                                       GFS3_OP_RELEASE,
+                                       client3_3_release_cbk, NULL,
+                                       NULL, 0, NULL, 0, NULL,
+                                       (xdrproc_t)xdr_gfs3_release_req);
         }
 
+        rpc_clnt_unref (conf->rpc);
 out:
-        if (!ret && fdctx) {
+        if (fdctx) {
                 fdctx->remote_fd = -1;
-                inode_unref (fdctx->inode);
                 GF_FREE (fdctx);
         }
 
@@ -2342,19 +2955,16 @@ out:
 }
 
 int32_t
-client3_1_releasedir (call_frame_t *frame, xlator_t *this,
+client3_3_releasedir (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
         clnt_conf_t         *conf        = NULL;
         clnt_fd_ctx_t       *fdctx       = NULL;
         clnt_args_t         *args        = NULL;
-        gfs3_releasedir_req  req         = {{0,},};
         int64_t              remote_fd   = -1;
-        int                  ret         = 0;
-        char                 parent_down = 0;
 
-        if (!frame || !this || !data)
-                goto unwind;
+        if (!this || !data)
+                goto out;
 
         args = data;
         conf = this->private;
@@ -2378,55 +2988,25 @@ client3_1_releasedir (call_frame_t *frame, xlator_t *this,
         }
         pthread_mutex_unlock (&conf->lock);
 
-        if (remote_fd != -1) {
-                pthread_mutex_lock (&conf->lock);
-                {
-                        parent_down = conf->parent_down;
-                        if (!parent_down) {
-                                rpc_clnt_ref (conf->rpc);
-                        }
-                }
-                pthread_mutex_unlock (&conf->lock);
-
-                if (!parent_down) {
-                        req.fd = remote_fd;
-
-                        ret = client_submit_request (this, &req, frame,
-                                                     conf->fops,
-                                                     GFS3_OP_RELEASEDIR,
-                                                     client3_1_releasedir_cbk,
-                                                     NULL, NULL, 0, NULL, 0,
-                                                     NULL,
-                                                     (xdrproc_t)xdr_gfs3_releasedir_req);
-
-                        rpc_clnt_unref (conf->rpc);
-                }
-
-                inode_unref (fdctx->inode);
-                GF_FREE (fdctx);
-        }
-
-unwind:
-        if (ret)
-                STACK_DESTROY (frame->root);
+        if (remote_fd != -1)
+                client_fdctx_destroy (this, fdctx);
+out:
 
         return 0;
 }
 
 int32_t
-client3_1_release (call_frame_t *frame, xlator_t *this,
+client3_3_release (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
-        int64_t           remote_fd   = -1;
-        clnt_conf_t      *conf        = NULL;
-        clnt_fd_ctx_t    *fdctx       = NULL;
-        clnt_args_t      *args        = NULL;
-        gfs3_release_req  req         = {{0,},};
-        int               ret         = 0;
-        char              parent_down = 0;
+        int64_t           remote_fd     = -1;
+        clnt_conf_t      *conf          = NULL;
+        clnt_fd_ctx_t    *fdctx         = NULL;
+        clnt_args_t      *args          = NULL;
+        lk_heal_state_t   lk_heal_state = GF_LK_HEAL_DONE;
 
-        if (!frame || !this || !data)
-                goto unwind;
+        if (!this || !data)
+                goto out;
 
         args = data;
         conf = this->private;
@@ -2435,14 +3015,16 @@ client3_1_release (call_frame_t *frame, xlator_t *this,
         {
                 fdctx = this_fd_del_ctx (args->fd, this);
                 if (fdctx != NULL) {
-                        remote_fd = fdctx->remote_fd;
+                        remote_fd     = fdctx->remote_fd;
+                        lk_heal_state = fdctx->lk_heal_state;
 
                         /* fdctx->remote_fd == -1 indicates a reopen attempt
                            in progress. Just mark ->released = 1 and let
                            reopen_cbk handle releasing
                         */
 
-                        if (remote_fd != -1)
+                        if (remote_fd != -1 &&
+                            lk_heal_state == GF_LK_HEAL_DONE)
                                 list_del_init (&fdctx->sfd_pos);
 
                         fdctx->released = 1;
@@ -2450,44 +3032,15 @@ client3_1_release (call_frame_t *frame, xlator_t *this,
         }
         pthread_mutex_unlock (&conf->lock);
 
-        if (remote_fd != -1) {
-                req.fd = remote_fd;
-
-                delete_granted_locks_fd (fdctx);
-
-                pthread_mutex_lock (&conf->lock);
-                {
-                        parent_down = conf->parent_down;
-                        if (!parent_down) {
-                                rpc_clnt_ref (conf->rpc);
-                        }
-                }
-                pthread_mutex_unlock (&conf->lock);
-
-                if (!parent_down) {
-                        ret = client_submit_request (this, &req, frame,
-                                                     conf->fops,
-                                                     GFS3_OP_RELEASE,
-                                                     client3_1_release_cbk,
-                                                     NULL, NULL,
-                                                     0, NULL, 0, NULL,
-                                                     (xdrproc_t)xdr_gfs3_release_req);
-                        rpc_clnt_unref (conf->rpc);
-                }
-
-                inode_unref (fdctx->inode);
-                GF_FREE (fdctx);
-        }
-unwind:
-        if (ret)
-                STACK_DESTROY (frame->root);
-
+        if (remote_fd != -1 && lk_heal_state == GF_LK_HEAL_DONE)
+                client_fdctx_destroy (this, fdctx);
+out:
         return 0;
 }
 
 
 int32_t
-client3_1_lookup (call_frame_t *frame, xlator_t *this,
+client3_3_lookup (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf              = NULL;
@@ -2515,12 +3068,13 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto unwind;
         }
+        frame->local = local;
 
         if (!(args->loc && args->loc->inode))
                 goto unwind;
 
         loc_copy (&local->loc, args->loc);
-        frame->local = local;
+        loc_path (&local->loc, NULL);
 
         if (args->loc->parent) {
                 if (!uuid_is_null (args->loc->parent->gfid))
@@ -2534,8 +3088,8 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
                         memcpy (req.gfid, args->loc->gfid, 16);
         }
 
-        if (args->dict) {
-                content = dict_get (args->dict, GF_CONTENT_KEY);
+        if (args->xdata) {
+                content = dict_get (args->xdata, GF_CONTENT_KEY);
                 if (content != NULL) {
                         rsp_iobref = iobref_new ();
                         if (rsp_iobref == NULL) {
@@ -2554,75 +3108,61 @@ client3_1_lookup (call_frame_t *frame, xlator_t *this,
                         iobuf_unref (rsp_iobuf);
                         rsphdr = &vector[0];
                         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-                        rsphdr->iov_len
-                                = iobuf_pagesize (rsp_iobuf);
+                        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
                         count = 1;
-                        rsp_iobuf = NULL;
                         local->iobref = rsp_iobref;
+                        rsp_iobuf = NULL;
                         rsp_iobref = NULL;
                 }
 
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata,
+                                            (&req.xdata.xdata_val),
+                                            req.xdata.xdata_len,
                                             op_errno, unwind);
         }
 
         if (args->loc->name)
-                req.bname         = (char *)args->loc->name;
+                req.bname = (char *)args->loc->name;
         else
                 req.bname = "";
 
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_LOOKUP, client3_1_lookup_cbk,
+                                     GFS3_OP_LOOKUP, client3_3_lookup_cbk,
                                      NULL, rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_lookup_req);
 
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.xdata.xdata_val);
 
-        if (rsp_iobref != NULL) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
 
         return 0;
 
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-
-        if (frame)
-                frame->local = NULL;
-
-        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+        CLIENT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
                              NULL);
 
-        client_local_wipe (local);
+        GF_FREE (req.xdata.xdata_val);
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
-
-        if (rsp_iobref != NULL) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        if (rsp_iobuf != NULL) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
         return 0;
 }
 
-
-
 int32_t
-client3_1_stat (call_frame_t *frame, xlator_t *this,
+client3_3_stat (call_frame_t *frame, xlator_t *this,
                 void *data)
 {
         clnt_conf_t   *conf     = NULL;
@@ -2648,26 +3188,31 @@ client3_1_stat (call_frame_t *frame, xlator_t *this,
                                        unwind, op_errno, EINVAL);
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_STAT, client3_1_stat_cbk, NULL,
+                                     GFS3_OP_STAT, client3_3_stat_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_stat_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop %s",
-                strerror (op_errno));
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_truncate (call_frame_t *frame, xlator_t *this,
+client3_3_truncate (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_conf_t       *conf     = NULL;
@@ -2696,25 +3241,31 @@ client3_1_truncate (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_TRUNCATE,
-                                     client3_1_truncate_cbk, NULL,
+                                     client3_3_truncate_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_truncate_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_ftruncate (call_frame_t *frame, xlator_t *this,
+client3_3_ftruncate (call_frame_t *frame, xlator_t *this,
                      void *data)
 {
         clnt_args_t        *args     = NULL;
@@ -2731,32 +3282,39 @@ client3_1_ftruncate (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.offset = args->offset;
         req.fd     = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FTRUNCATE,
-                                     client3_1_ftruncate_cbk, NULL,
+                                     client3_3_ftruncate_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_ftruncate_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_access (call_frame_t *frame, xlator_t *this,
+client3_3_access (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf     = NULL;
@@ -2785,32 +3343,43 @@ client3_1_access (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_ACCESS,
-                                     client3_1_access_cbk, NULL,
+                                     client3_3_access_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_access_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (access, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 int32_t
-client3_1_readlink (call_frame_t *frame, xlator_t *this,
+client3_3_readlink (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
-        clnt_conf_t       *conf     = NULL;
-        clnt_args_t       *args     = NULL;
-        gfs3_readlink_req  req      = {{0,},};
-        int                ret      = 0;
-        int                op_errno = ESTALE;
+        clnt_conf_t       *conf              = NULL;
+        clnt_args_t       *args              = NULL;
+        gfs3_readlink_req  req               = {{0,},};
+        int                ret               = 0;
+        int                op_errno          = ESTALE;
+        clnt_local_t      *local             = NULL;
+        struct iobuf      *rsp_iobuf         = NULL;
+        struct iobref     *rsp_iobref        = NULL;
+        struct iovec      *rsphdr            = NULL;
+        int                count             = 0;
+        struct iovec       vector[MAX_IOVEC] = {{0}, };
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -2831,20 +3400,58 @@ client3_1_readlink (call_frame_t *frame, xlator_t *this,
         req.size = args->size;
         conf = this->private;
 
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        frame->local = local;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        rsp_iobref = iobref_new ();
+        if (rsp_iobref == NULL) {
+                goto unwind;
+        }
+
+        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
+        if (rsp_iobuf == NULL) {
+                goto unwind;
+        }
+
+        iobref_add (rsp_iobref, rsp_iobuf);
+        iobuf_unref (rsp_iobuf);
+        rsphdr = &vector[0];
+        rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
+        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
+        count = 1;
+        local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
+        rsp_iobref = NULL;
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_READLINK,
-                                     client3_1_readlink_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_readlink_req);
+                                     client3_3_readlink_cbk, NULL,
+                                     rsphdr, count, NULL, 0,
+                                     local->iobref,
+                                     (xdrproc_t)xdr_gfs3_readlink_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL);
+        if (rsp_iobref != NULL) {
+                iobref_unref (rsp_iobref);
+        }
+
+        CLIENT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
@@ -2852,7 +3459,7 @@ unwind:
 
 
 int32_t
-client3_1_unlink (call_frame_t *frame, xlator_t *this,
+client3_3_unlink (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf     = NULL;
@@ -2880,26 +3487,32 @@ client3_1_unlink (call_frame_t *frame, xlator_t *this,
         req.bname = (char *)args->loc->name;
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_UNLINK,
-                                     client3_1_unlink_cbk, NULL,
+                                     client3_3_unlink_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_unlink_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_rmdir (call_frame_t *frame, xlator_t *this,
+client3_3_rmdir (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_conf_t    *conf     = NULL;
@@ -2925,28 +3538,33 @@ client3_1_rmdir (call_frame_t *frame, xlator_t *this,
                                        !uuid_is_null (*((uuid_t*)req.pargfid)),
                                        unwind, op_errno, EINVAL);
         req.bname = (char *)args->loc->name;
-        req.flags = args->flags;
+        req.xflags = args->flags;
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_RMDIR, client3_1_rmdir_cbk, NULL,
+                                     GFS3_OP_RMDIR, client3_3_rmdir_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_rmdir_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_symlink (call_frame_t *frame, xlator_t *this,
+client3_3_symlink (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_local_t     *local    = NULL;
@@ -2966,11 +3584,13 @@ client3_1_symlink (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
+        frame->local = local;
+
         if (!(args->loc && args->loc->parent))
                 goto unwind;
 
         loc_copy (&local->loc, args->loc);
-        frame->local = local;
+        loc_path (&local->loc, NULL);
 
         if (!uuid_is_null (args->loc->parent->gfid))
                 memcpy (req.pargfid,  args->loc->parent->gfid, 16);
@@ -2982,42 +3602,39 @@ client3_1_symlink (call_frame_t *frame, xlator_t *this,
                                        unwind, op_errno, EINVAL);
         req.linkname = (char *)args->linkname;
         req.bname    = (char *)args->loc->name;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->dict, (&req.dict.dict_val),
-                                    req.dict.dict_len, op_errno, unwind);
+        req.umask = args->umask;
+        local->loc2.path = gf_strdup (req.linkname);
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_SYMLINK, client3_1_symlink_cbk,
+                                     GFS3_OP_SYMLINK, client3_3_symlink_cbk,
                                      NULL,  NULL, 0, NULL,
                                      0, NULL, (xdrproc_t)xdr_gfs3_symlink_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
 
-        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        CLIENT_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
 
-        client_local_wipe (local);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
         return 0;
 }
 
 
 
 int32_t
-client3_1_rename (call_frame_t *frame, xlator_t *this,
+client3_3_rename (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf     = NULL;
@@ -3055,25 +3672,33 @@ client3_1_rename (call_frame_t *frame, xlator_t *this,
         req.newbname = (char *)args->newloc->name;
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_RENAME, client3_1_rename_cbk, NULL,
+                                     GFS3_OP_RENAME, client3_3_rename_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_rename_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        CLIENT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_link (call_frame_t *frame, xlator_t *this,
+client3_3_link (call_frame_t *frame, xlator_t *this,
                 void *data)
 {
         clnt_local_t  *local    = NULL;
@@ -3115,30 +3740,39 @@ client3_1_link (call_frame_t *frame, xlator_t *this,
         }
 
         loc_copy (&local->loc, args->oldloc);
+        loc_path (&local->loc, NULL);
+        loc_copy (&local->loc2, args->newloc);
+        loc_path (&local->loc2, NULL);
         frame->local = local;
 
         req.newbname = (char *)args->newloc->name;
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_LINK, client3_1_link_cbk, NULL,
+                                     GFS3_OP_LINK, client3_3_link_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_link_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        CLIENT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_mknod (call_frame_t *frame, xlator_t *this,
+client3_3_mknod (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_local_t   *local    = NULL;
@@ -3158,12 +3792,13 @@ client3_1_mknod (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto unwind;
         }
+        frame->local = local;
 
         if (!(args->loc && args->loc->parent))
                 goto unwind;
 
         loc_copy (&local->loc, args->loc);
-        frame->local = local;
+        loc_path (&local->loc, NULL);
 
         if (!uuid_is_null (args->loc->parent->gfid))
                 memcpy (req.pargfid,  args->loc->parent->gfid, 16);
@@ -3176,45 +3811,36 @@ client3_1_mknod (call_frame_t *frame, xlator_t *this,
         req.bname  = (char *)args->loc->name;
         req.mode   = args->mode;
         req.dev    = args->rdev;
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
+        req.umask = args->umask;
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_MKNOD, client3_1_mknod_cbk, NULL,
+                                     GFS3_OP_MKNOD, client3_3_mknod_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_mknod_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
+        CLIENT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL);
 
-        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
 
-        client_local_wipe (local);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
         return 0;
 }
 
 
 
 int32_t
-client3_1_mkdir (call_frame_t *frame, xlator_t *this,
+client3_3_mkdir (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_local_t   *local    = NULL;
@@ -3234,12 +3860,13 @@ client3_1_mkdir (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto unwind;
         }
+        frame->local = local;
 
         if (!(args->loc && args->loc->parent))
                 goto unwind;
 
         loc_copy (&local->loc, args->loc);
-        frame->local = local;
+        loc_path (&local->loc, NULL);
 
         if (!uuid_is_null (args->loc->parent->gfid))
                 memcpy (req.pargfid,  args->loc->parent->gfid, 16);
@@ -3252,44 +3879,35 @@ client3_1_mkdir (call_frame_t *frame, xlator_t *this,
 
         req.bname = (char *)args->loc->name;
         req.mode  = args->mode;
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
+        req.umask = args->umask;
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_MKDIR, client3_1_mkdir_cbk, NULL,
+                                     GFS3_OP_MKDIR, client3_3_mkdir_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_mkdir_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
+        CLIENT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL);
 
-        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
 
-        client_local_wipe (local);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
         return 0;
 }
 
 
 int32_t
-client3_1_create (call_frame_t *frame, xlator_t *this,
+client3_3_create (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_local_t    *local    = NULL;
@@ -3309,6 +3927,8 @@ client3_1_create (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto unwind;
         }
+        frame->local = local;
+
         if (!(args->loc && args->loc->parent))
                 goto unwind;
 
@@ -3316,7 +3936,7 @@ client3_1_create (call_frame_t *frame, xlator_t *this,
         local->flags = args->flags;
 
         loc_copy (&local->loc, args->loc);
-        frame->local = local;
+        loc_path (&local->loc, NULL);
 
         if (!uuid_is_null (args->loc->parent->gfid))
                 memcpy (req.pargfid,  args->loc->parent->gfid, 16);
@@ -3329,45 +3949,37 @@ client3_1_create (call_frame_t *frame, xlator_t *this,
         req.bname = (char *)args->loc->name;
         req.mode  = args->mode;
         req.flags = gf_flags_from_flags (args->flags);
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
+        req.umask = args->umask;
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_CREATE, client3_1_create_cbk, NULL,
+                                     GFS3_OP_CREATE, client3_3_create_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_create_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
+        CLIENT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
 
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
-        client_local_wipe (local);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
         return 0;
 }
 
 
 
 int32_t
-client3_1_open (call_frame_t *frame, xlator_t *this,
+client3_3_open (call_frame_t *frame, xlator_t *this,
                 void *data)
 {
         clnt_local_t  *local    = NULL;
@@ -3387,14 +3999,15 @@ client3_1_open (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto unwind;
         }
+        frame->local = local;
+
         if (!(args->loc && args->loc->inode))
                 goto unwind;
 
         local->fd = fd_ref (args->fd);
         local->flags = args->flags;
-        local->wbflags = args->wbflags;
         loc_copy (&local->loc, args->loc);
-        frame->local = local;
+        loc_path (&local->loc, NULL);
 
         if (!uuid_is_null (args->loc->inode->gfid))
                 memcpy (req.gfid,  args->loc->inode->gfid, 16);
@@ -3405,46 +4018,47 @@ client3_1_open (call_frame_t *frame, xlator_t *this,
                                        !uuid_is_null (*((uuid_t*)req.gfid)),
                                        unwind, op_errno, EINVAL);
         req.flags = gf_flags_from_flags (args->flags);
-        req.wbflags = args->wbflags;
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPEN, client3_1_open_cbk, NULL,
+                                     GFS3_OP_OPEN, client3_3_open_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_open_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
+        CLIENT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
 
-        STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
 
-        client_local_wipe (local);
         return 0;
 }
 
 
 
 int32_t
-client3_1_readv (call_frame_t *frame, xlator_t *this,
+client3_3_readv (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_args_t    *args       = NULL;
         int64_t         remote_fd  = -1;
         clnt_conf_t    *conf       = NULL;
+        clnt_local_t   *local      = NULL;
         int             op_errno   = ESTALE;
         gfs3_read_req   req        = {{0,},};
         int             ret        = 0;
         struct iovec    rsp_vec    = {0, };
         struct iobuf   *rsp_iobuf  = NULL;
         struct iobref  *rsp_iobref = NULL;
-        clnt_local_t   *local      = NULL;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3452,7 +4066,14 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
+        local = frame->local;
 
         req.size   = args->size;
         req.offset = args->offset;
@@ -3475,6 +4096,7 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
 
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
+
         rsp_vec.iov_base = iobuf_ptr (rsp_iobuf);
         rsp_vec.iov_len = iobuf_pagesize (rsp_iobuf);
 
@@ -3489,51 +4111,48 @@ client3_1_readv (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
         local->iobref = rsp_iobref;
         rsp_iobref = NULL;
-        frame->local = local;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
 
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_READ, client3_1_readv_cbk, NULL,
+                                     GFS3_OP_READ, client3_3_readv_cbk, NULL,
                                      NULL, 0, &rsp_vec, 1,
                                      local->iobref,
                                      (xdrproc_t)xdr_gfs3_read_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                //unwind is done in the cbk
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        CLIENT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_writev (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_writev (call_frame_t *frame, xlator_t *this, void *data)
 {
         clnt_args_t    *args     = NULL;
         int64_t         remote_fd = -1;
         clnt_conf_t    *conf     = NULL;
         gfs3_write_req  req      = {{0,},};
         int             op_errno = ESTALE;
-        int             ret        = 0;
+        int             ret      = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3541,7 +4160,13 @@ client3_1_writev (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
 
         req.size   = args->size;
         req.offset = args->offset;
@@ -3550,9 +4175,21 @@ client3_1_writev (call_frame_t *frame, xlator_t *this, void *data)
 
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        ret = client_submit_vec_request (this, &req, frame, conf->fops, GFS3_OP_WRITE,
-                                         client3_1_writev_cbk, args->vector,
-                                         args->count, args->iobref,
+#ifdef GF_TESTING_IO_XDATA
+        if (!args->xdata)
+                args->xdata = dict_new ();
+
+        ret = dict_set_str (args->xdata, "testing-the-xdata-key",
+                            "testing-the-xdata-value");
+#endif
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_vec_request (this, &req, frame, conf->fops,
+                                         GFS3_OP_WRITE, client3_3_writev_cbk,
+                                         args->vector, args->count,
+                                         args->iobref,
                                          (xdrproc_t)xdr_gfs3_write_req);
         if (ret) {
                 /*
@@ -3560,20 +4197,23 @@ client3_1_writev (call_frame_t *frame, xlator_t *this, void *data)
                  * do the unwind for us (see rpc_clnt_submit), so don't unwind
                  * here in such cases.
                  */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to send the fop: %s", strerror (op_errno));
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_flush (call_frame_t *frame, xlator_t *this,
+client3_3_flush (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_args_t    *args     = NULL;
@@ -3590,15 +4230,15 @@ client3_1_flush (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         conf = this->private;
 
         local = mem_get0 (this->local_pool);
         if (!local) {
-                STACK_UNWIND (frame, -1, ENOMEM);
-                return 0;
-
+                op_errno = ENOMEM;
+                goto unwind;
         }
 
         local->fd = fd_ref (args->fd);
@@ -3608,33 +4248,41 @@ client3_1_flush (call_frame_t *frame, xlator_t *this,
         req.fd = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FLUSH, client3_1_flush_cbk, NULL,
+                                     GFS3_OP_FLUSH, client3_3_flush_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_flush_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fsync (call_frame_t *frame, xlator_t *this,
+client3_3_fsync (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
-        clnt_args_t    *args     = NULL;
-        gfs3_fsync_req  req      = {{0,},};
+        clnt_args_t    *args      = NULL;
+        gfs3_fsync_req  req       = {{0,},};
         int64_t         remote_fd = -1;
-        clnt_conf_t    *conf     = NULL;
-        int             op_errno = 0;
-        int           ret        = 0;
+        clnt_conf_t    *conf      = NULL;
+        int             op_errno  = 0;
+        int             ret       = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3642,32 +4290,40 @@ client3_1_fsync (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd   = remote_fd;
         req.data = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
 
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSYNC, client3_1_fsync_cbk, NULL,
+                                     GFS3_OP_FSYNC, client3_3_fsync_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_fsync_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fstat (call_frame_t *frame, xlator_t *this,
+client3_3_fstat (call_frame_t *frame, xlator_t *this,
                  void *data)
 {
         clnt_args_t    *args     = NULL;
@@ -3683,30 +4339,38 @@ client3_1_fstat (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSTAT, client3_1_fstat_cbk, NULL,
+                                     GFS3_OP_FSTAT, client3_3_fstat_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_fstat_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_opendir (call_frame_t *frame, xlator_t *this,
+client3_3_opendir (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_local_t     *local    = NULL;
@@ -3726,12 +4390,14 @@ client3_1_opendir (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto unwind;
         }
+        frame->local = local;
+
         if (!(args->loc && args->loc->inode))
                 goto unwind;
 
         local->fd = fd_ref (args->fd);
         loc_copy (&local->loc, args->loc);
-        frame->local = local;
+        loc_path (&local->loc, NULL);
 
         if (!uuid_is_null (args->loc->inode->gfid))
                 memcpy (req.gfid,  args->loc->inode->gfid, 16);
@@ -3744,35 +4410,40 @@ client3_1_opendir (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPENDIR, client3_1_opendir_cbk,
+                                     GFS3_OP_OPENDIR, client3_3_opendir_cbk,
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_opendir_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        if (frame)
-                frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL);
-        client_local_wipe (local);
+        CLIENT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args     = NULL;
+        clnt_args_t       *args      = NULL;
         int64_t            remote_fd = -1;
-        clnt_conf_t       *conf     = NULL;
-        int                op_errno = ESTALE;
-        gfs3_fsyncdir_req  req      = {{0,},};
-        int                ret        = 0;
+        clnt_conf_t       *conf      = NULL;
+        gfs3_fsyncdir_req  req       = {{0,},};
+        int                ret       = 0;
+        int32_t            op_errno  = ESTALE;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -3780,7 +4451,8 @@ client3_1_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd   = remote_fd;
         req.data = args->flags;
@@ -3788,26 +4460,33 @@ client3_1_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSYNCDIR, client3_1_fsyncdir_cbk,
+                                     GFS3_OP_FSYNCDIR, client3_3_fsyncdir_cbk,
                                      NULL, NULL, 0,
                                      NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_fsyncdir_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_statfs (call_frame_t *frame, xlator_t *this,
+client3_3_statfs (call_frame_t *frame, xlator_t *this,
                   void *data)
 {
         clnt_conf_t     *conf     = NULL;
@@ -3838,25 +4517,32 @@ client3_1_statfs (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_STATFS, client3_1_statfs_cbk, NULL,
+                                     GFS3_OP_STATFS, client3_3_statfs_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_statfs_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
+
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_setxattr (call_frame_t *frame, xlator_t *this,
+client3_3_setxattr (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_conf_t       *conf     = NULL;
@@ -3881,8 +4567,8 @@ client3_1_setxattr (call_frame_t *frame, xlator_t *this,
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
                                        !uuid_is_null (*((uuid_t*)req.gfid)),
                                        unwind, op_errno, EINVAL);
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
+        if (args->xattr) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
                                             (&req.dict.dict_val),
                                             req.dict.dict_len,
                                             op_errno, unwind);
@@ -3892,32 +4578,34 @@ client3_1_setxattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_SETXATTR, client3_1_setxattr_cbk,
+                                     GFS3_OP_SETXATTR, client3_3_setxattr_cbk,
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_setxattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
-        }
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        CLIENT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fsetxattr (call_frame_t *frame, xlator_t *this,
+client3_3_fsetxattr (call_frame_t *frame, xlator_t *this,
                      void *data)
 {
         clnt_args_t        *args     = NULL;
@@ -3933,39 +4621,42 @@ client3_1_fsetxattr (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd    = remote_fd;
         req.flags = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
+        if (args->xattr) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
                                             (&req.dict.dict_val),
                                             req.dict.dict_len,
                                             op_errno, unwind);
         }
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSETXATTR, client3_1_fsetxattr_cbk,
+                                     GFS3_OP_FSETXATTR, client3_3_fsetxattr_cbk,
                                      NULL, NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_fsetxattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno);
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        CLIENT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
@@ -3973,7 +4664,7 @@ unwind:
 
 
 int32_t
-client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
+client3_3_fgetxattr (call_frame_t *frame, xlator_t *this,
                      void *data)
 {
         clnt_args_t        *args       = NULL;
@@ -3995,7 +4686,8 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         local = mem_get0 (this->local_pool);
         if (!local) {
@@ -4019,12 +4711,13 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
 
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
+
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
         rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);;
         count = 1;
-        rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
         rsp_iobref = NULL;
 
         req.namelen = 1; /* Use it as a flag */
@@ -4036,34 +4729,38 @@ client3_1_fgetxattr (call_frame_t *frame, xlator_t *this,
         }
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FGETXATTR,
-                                     client3_1_fgetxattr_cbk, NULL,
+                                     client3_3_fgetxattr_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_fgetxattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        return 0;
-unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
+        GF_FREE (req.xdata.xdata_val);
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
 
-        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL);
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
 
-        client_local_wipe (local);
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 }
@@ -4071,7 +4768,7 @@ unwind:
 
 
 int32_t
-client3_1_getxattr (call_frame_t *frame, xlator_t *this,
+client3_3_getxattr (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_conf_t       *conf       = NULL;
@@ -4094,7 +4791,7 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
         }
         args = data;
 
-        if (!(args->loc && args->loc->inode)) {
+        if (!args->loc) {
                 op_errno = EINVAL;
                 goto unwind;
         }
@@ -4104,6 +4801,12 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto unwind;
         }
+
+        loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
+        if (args->name)
+                local->name = gf_strdup (args->name);
+
         frame->local = local;
 
         rsp_iobref = iobref_new ();
@@ -4121,15 +4824,16 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
 
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
+
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
         rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
-        rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
         rsp_iobref = NULL;
 
-        if (!uuid_is_null (args->loc->inode->gfid))
+        if (args->loc->inode && !uuid_is_null (args->loc->inode->gfid))
                 memcpy (req.gfid,  args->loc->inode->gfid, 16);
         else
                 memcpy (req.gfid, args->loc->gfid, 16);
@@ -4166,34 +4870,38 @@ client3_1_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_GETXATTR,
-                                     client3_1_getxattr_cbk, NULL,
+                                     client3_3_getxattr_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_getxattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s",
-                strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
-        client_local_wipe (local);
-
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        CLIENT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 }
@@ -4201,7 +4909,7 @@ unwind:
 
 
 int32_t
-client3_1_xattrop (call_frame_t *frame, xlator_t *this,
+client3_3_xattrop (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_conf_t      *conf       = NULL;
@@ -4246,12 +4954,13 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
 
         iobref_add (rsp_iobref, rsp_iobuf);
         iobuf_unref (rsp_iobuf);
+
         rsphdr = &vector[0];
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
         rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
-        rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
         rsp_iobref = NULL;
 
         if (!uuid_is_null (args->loc->inode->gfid))
@@ -4259,11 +4968,13 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
         else
                 memcpy (req.gfid, args->loc->gfid, 16);
 
+        loc_copy (&local->loc, args->loc);
+        loc_path (&local->loc, NULL);
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
                                        !uuid_is_null (*((uuid_t*)req.gfid)),
                                        unwind, op_errno, EINVAL);
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
+        if (args->xattr) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
                                             (&req.dict.dict_val),
                                             req.dict.dict_len,
                                             op_errno, unwind);
@@ -4273,41 +4984,42 @@ client3_1_xattrop (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_XATTROP,
-                                     client3_1_xattrop_cbk, NULL,
+                                     client3_3_xattrop_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_xattrop_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
-        return 0;
-unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
+        GF_FREE (req.dict.dict_val);
 
-        STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
 
-        client_local_wipe (local);
+        GF_FREE (req.dict.dict_val);
 
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 }
@@ -4315,17 +5027,17 @@ unwind:
 
 
 int32_t
-client3_1_fxattrop (call_frame_t *frame, xlator_t *this,
+client3_3_fxattrop (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_args_t       *args       = NULL;
         int64_t            remote_fd  = -1;
         clnt_conf_t       *conf       = NULL;
+        clnt_local_t      *local      = NULL;
         gfs3_fxattrop_req  req        = {{0,},};
         int                op_errno   = ESTALE;
         int                ret        = 0;
         int                count      = 0;
-        clnt_local_t    *local      = NULL;
         struct iobref     *rsp_iobref = NULL;
         struct iobuf      *rsp_iobuf  = NULL;
         struct iovec      *rsphdr     = NULL;
@@ -4337,19 +5049,20 @@ client3_1_fxattrop (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
+
+        local = frame->local;
 
         req.fd     = remote_fd;
         req.flags  = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
         rsp_iobref = iobref_new ();
         if (rsp_iobref == NULL) {
                 op_errno = ENOMEM;
@@ -4369,61 +5082,54 @@ client3_1_fxattrop (call_frame_t *frame, xlator_t *this,
         rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
         rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
         count = 1;
-        rsp_iobuf = NULL;
         local->iobref = rsp_iobref;
+        rsp_iobuf = NULL;
         rsp_iobref = NULL;
 
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
+        if (args->xattr) {
+                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
                                             (&req.dict.dict_val),
                                             req.dict.dict_len,
                                             op_errno, unwind);
         }
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FXATTROP,
-                                     client3_1_fxattrop_cbk, NULL,
+                                     client3_3_fxattrop_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, local->iobref,
                                      (xdrproc_t)xdr_gfs3_fxattrop_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
+        GF_FREE (req.dict.dict_val);
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
+        CLIENT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
 
-        STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL);
+        GF_FREE (req.dict.dict_val);
 
-        if (req.dict.dict_val) {
-                GF_FREE (req.dict.dict_val);
-        }
-
-        client_local_wipe (local);
-
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
+
+        GF_FREE (req.xdata.xdata_val);
 
         return 0;
 }
 
 
-
 int32_t
-client3_1_removexattr (call_frame_t *frame, xlator_t *this,
+client3_3_removexattr (call_frame_t *frame, xlator_t *this,
                        void *data)
 {
         clnt_conf_t          *conf     = NULL;
@@ -4452,25 +5158,30 @@ client3_1_removexattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_REMOVEXATTR,
-                                     client3_1_removexattr_cbk, NULL,
+                                     client3_3_removexattr_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_removexattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 int32_t
-client3_1_fremovexattr (call_frame_t *frame, xlator_t *this,
+client3_3_fremovexattr (call_frame_t *frame, xlator_t *this,
                         void *data)
 {
         clnt_conf_t           *conf     = NULL;
@@ -4490,31 +5201,37 @@ client3_1_fremovexattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         memcpy (req.gfid,  args->fd->inode->gfid, 16);
         req.name = (char *)args->name;
         req.fd = remote_fd;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FREMOVEXATTR,
-                                     client3_1_fremovexattr_cbk, NULL,
+                                     client3_3_fremovexattr_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_fremovexattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 int32_t
-client3_1_lk (call_frame_t *frame, xlator_t *this,
+client3_3_lk (call_frame_t *frame, xlator_t *this,
               void *data)
 {
         clnt_args_t     *args       = NULL;
@@ -4537,8 +5254,10 @@ client3_1_lk (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto unwind;
         }
+        frame->local = local;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         ret = client_cmd_to_gf_cmd (args->cmd, &gf_cmd);
         if (ret) {
@@ -4563,7 +5282,6 @@ client3_1_lk (call_frame_t *frame, xlator_t *this,
         local->owner = frame->root->lk_owner;
         local->cmd   = args->cmd;
         local->fd    = fd_ref (args->fd);
-        frame->local = local;
 
         req.fd    = remote_fd;
         req.cmd   = gf_cmd;
@@ -4572,25 +5290,30 @@ client3_1_lk (call_frame_t *frame, xlator_t *this,
 
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops, GFS3_OP_LK,
-                                     client3_1_lk_cbk, NULL,
+                                     client3_3_lk_cbk, NULL,
                                      NULL, 0, NULL, 0, NULL,
                                      (xdrproc_t)xdr_gfs3_lk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_inodelk (call_frame_t *frame, xlator_t *this,
+client3_3_inodelk (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_conf_t      *conf     = NULL;
@@ -4648,27 +5371,32 @@ client3_1_inodelk (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_INODELK,
-                                     client3_1_inodelk_cbk, NULL,
+                                     client3_3_inodelk_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_inodelk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_finodelk (call_frame_t *frame, xlator_t *this,
+client3_3_finodelk (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_args_t       *args     = NULL;
@@ -4678,15 +5406,20 @@ client3_1_finodelk (call_frame_t *frame, xlator_t *this,
         int64_t            remote_fd = -1;
         clnt_conf_t       *conf     = NULL;
         int                op_errno = ESTALE;
-        int           ret        = 0;
+        int                ret      = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
 
         args = data;
         conf = this->private;
-
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
+                              remote_fd, op_errno, unwind);
+        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
+        if (ret) {
+                op_errno = -ret;
+                goto unwind;
+        }
 
         if (args->cmd == F_GETLK || args->cmd == F_GETLK64)
                 gf_cmd = GF_LK_GETLK;
@@ -4719,26 +5452,30 @@ client3_1_finodelk (call_frame_t *frame, xlator_t *this,
         gf_proto_flock_from_flock (&req.flock, args->flock);
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FINODELK,
-                                     client3_1_finodelk_cbk, NULL,
+                                     client3_3_finodelk_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_finodelk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_entrylk (call_frame_t *frame, xlator_t *this,
+client3_3_entrylk (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_conf_t      *conf     = NULL;
@@ -4774,27 +5511,32 @@ client3_1_entrylk (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_ENTRYLK,
-                                     client3_1_entrylk_cbk, NULL,
+                                     client3_3_entrylk_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_entrylk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_fentrylk (call_frame_t *frame, xlator_t *this,
+client3_3_fentrylk (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_args_t       *args     = NULL;
@@ -4810,7 +5552,8 @@ client3_1_fentrylk (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd  = remote_fd;
         req.cmd = args->cmd_entrylk;
@@ -4823,26 +5566,31 @@ client3_1_fentrylk (call_frame_t *frame, xlator_t *this,
         }
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FENTRYLK,
-                                     client3_1_fentrylk_cbk, NULL,
+                                     client3_3_fentrylk_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_fentrylk_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno);
+        CLIENT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_rchecksum (call_frame_t *frame, xlator_t *this,
+client3_3_rchecksum (call_frame_t *frame, xlator_t *this,
                      void *data)
 {
         clnt_args_t        *args     = NULL;
@@ -4858,33 +5606,40 @@ client3_1_rchecksum (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.len    = args->len;
         req.offset = args->offset;
         req.fd     = remote_fd;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_RCHECKSUM,
-                                     client3_1_rchecksum_cbk, NULL,
+                                     client3_3_rchecksum_cbk, NULL,
                                      NULL, 0, NULL,
-                                     0, NULL, (xdrproc_t)xdr_gfs3_rchecksum_req);
+                                     0, NULL,
+                                     (xdrproc_t)xdr_gfs3_rchecksum_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (rchecksum, frame, -1, op_errno, 0, NULL);
+        CLIENT_STACK_UNWIND (rchecksum, frame, -1, op_errno, 0, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 
 int32_t
-client3_1_readdir (call_frame_t *frame, xlator_t *this,
+client3_3_readdir (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_args_t      *args       = NULL;
@@ -4908,7 +5663,8 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         readdir_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdir_rsp, &rsp)
                 + args->size;
@@ -4937,12 +5693,14 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
 
                 iobref_add (rsp_iobref, rsp_iobuf);
                 iobuf_unref (rsp_iobuf);
+
                 rsphdr = &vector[0];
                 rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
                 rsphdr->iov_len  = iobuf_pagesize (rsp_iobuf);
                 count = 1;
-                rsp_iobuf = NULL;
                 local->iobref = rsp_iobref;
+                rsp_iobuf = NULL;
+                rsp_iobref = NULL;
         }
 
         req.size = args->size;
@@ -4952,43 +5710,46 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
         local->cmd = remote_fd;
 
         memcpy (req.gfid, args->fd->inode->gfid, 16);
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
 
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_READDIR,
-                                     client3_1_readdir_cbk, NULL,
+                                     client3_3_readdir_cbk, NULL,
                                      rsphdr, count,
                                      NULL, 0, rsp_iobref,
                                      (xdrproc_t)xdr_gfs3_readdir_req);
-        rsp_iobref = NULL;
 
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
+
         return 0;
 
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
-        client_local_wipe (local);
-
-        if (rsp_iobref != NULL) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        if (rsp_iobuf != NULL) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 
 int32_t
-client3_1_readdirp (call_frame_t *frame, xlator_t *this,
+client3_3_readdirp (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
         clnt_args_t      *args              = NULL;
@@ -5012,7 +5773,8 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         readdirp_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdirp_rsp, &rsp)
                 + args->size;
@@ -5042,12 +5804,13 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
 
                 iobref_add (rsp_iobref, rsp_iobuf);
                 iobuf_unref (rsp_iobuf);
+
                 rsphdr = &vector[0];
                 rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
                 rsphdr->iov_len  = iobuf_pagesize (rsp_iobuf);
                 count = 1;
-                rsp_iobuf = NULL;
                 local->iobref = rsp_iobref;
+                rsp_iobuf = NULL;
                 rsp_iobref = NULL;
         }
 
@@ -5058,52 +5821,45 @@ client3_1_readdirp (call_frame_t *frame, xlator_t *this,
         req.fd = remote_fd;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
-        if (args->dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->dict,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
+        /* dict itself is 'xdata' here */
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.dict.dict_val),
+                                    req.dict.dict_len, op_errno, unwind);
 
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_READDIRP,
-                                     client3_1_readdirp_cbk, NULL,
+                                     client3_3_readdirp_cbk, NULL,
                                      rsphdr, count, NULL,
                                      0, rsp_iobref,
                                      (xdrproc_t)xdr_gfs3_readdirp_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
+
+        if (rsp_iobuf)
+                iobuf_unref (rsp_iobuf);
+
+        if (rsp_iobref)
+                iobref_unref (rsp_iobref);
 
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        local = frame->local;
-        frame->local = NULL;
-        client_local_wipe (local);
-
-        if (rsp_iobref) {
+        if (rsp_iobref)
                 iobref_unref (rsp_iobref);
-        }
 
-        if (rsp_iobuf) {
+        if (rsp_iobuf)
                 iobuf_unref (rsp_iobuf);
-        }
 
-        if (req.dict.dict_val)
-                GF_FREE (req.dict.dict_val);
+        GF_FREE (req.dict.dict_val);
 
-        STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL);
+        CLIENT_STACK_UNWIND (readdirp, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
-client3_1_setattr (call_frame_t *frame, xlator_t *this,
+client3_3_setattr (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
         clnt_conf_t      *conf     = NULL;
@@ -5133,25 +5889,30 @@ client3_1_setattr (call_frame_t *frame, xlator_t *this,
 
         conf = this->private;
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_SETATTR,
-                                     client3_1_setattr_cbk, NULL,
+                                     client3_3_setattr_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_setattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
-                goto unwind;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
 int32_t
-client3_1_fsetattr (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_fsetattr (call_frame_t *frame, xlator_t *this, void *data)
 {
         clnt_args_t       *args     = NULL;
         int64_t            remote_fd = -1;
@@ -5166,84 +5927,228 @@ client3_1_fsetattr (call_frame_t *frame, xlator_t *this, void *data)
         args = data;
         conf = this->private;
 
-        CLIENT_GET_REMOTE_FD(conf, args->fd, remote_fd, op_errno, unwind);
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
 
         req.fd = remote_fd;
         req.valid = args->valid;
         gf_stat_from_iatt (&req.stbuf, args->stbuf);
 
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FSETATTR,
-                                     client3_1_fsetattr_cbk, NULL,
+                                     client3_3_fsetattr_cbk, NULL,
                                      NULL, 0, NULL, 0,
                                      NULL, (xdrproc_t)xdr_gfs3_fsetattr_req);
         if (ret) {
-                op_errno = ENOTCONN;
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+        }
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
+
+int32_t
+client3_3_fallocate(call_frame_t *frame, xlator_t *this, void *data)
+{
+        clnt_args_t       *args     = NULL;
+        int64_t            remote_fd = -1;
+        clnt_conf_t       *conf     = NULL;
+	gfs3_fallocate_req req	    = {{0},};
+        int                op_errno = ESTALE;
+        int                ret        = 0;
+
+        if (!frame || !this || !data)
                 goto unwind;
+
+        args = data;
+        conf = this->private;
+
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
+
+        req.fd = remote_fd;
+	req.flags = args->flags;
+	req.offset = args->offset;
+	req.size = args->size;
+	memcpy(req.gfid, args->fd->inode->gfid, 16);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_request (this, &req, frame, conf->fops,
+                                     GFS3_OP_FALLOCATE,
+                                     client3_3_fallocate_cbk, NULL,
+                                     NULL, 0, NULL, 0,
+                                     NULL, (xdrproc_t)xdr_gfs3_fallocate_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
         }
 
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 unwind:
-        gf_log (this->name, GF_LOG_WARNING, "failed to send the fop: %s", strerror (op_errno));
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL);
+        CLIENT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
         return 0;
 }
 
+int32_t
+client3_3_discard(call_frame_t *frame, xlator_t *this, void *data)
+{
+        clnt_args_t       *args     = NULL;
+        int64_t            remote_fd = -1;
+        clnt_conf_t       *conf     = NULL;
+	gfs3_discard_req   req	    = {{0},};
+        int                op_errno = ESTALE;
+        int                ret        = 0;
+
+        if (!frame || !this || !data)
+                goto unwind;
 
+        args = data;
+        conf = this->private;
+
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
+
+        req.fd = remote_fd;
+	req.offset = args->offset;
+	req.size = args->size;
+	memcpy(req.gfid, args->fd->inode->gfid, 16);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_request(this, &req, frame, conf->fops,
+                                    GFS3_OP_DISCARD, client3_3_discard_cbk,
+				    NULL, NULL, 0, NULL, 0, NULL,
+                                    (xdrproc_t) xdr_gfs3_discard_req);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
+
+int32_t
+client3_3_zerofill(call_frame_t *frame, xlator_t *this, void *data)
+{
+        clnt_args_t       *args        = NULL;
+        int64_t            remote_fd   = -1;
+        clnt_conf_t       *conf        = NULL;
+        gfs3_zerofill_req   req        = {{0},};
+        int                op_errno    = ESTALE;
+        int                ret         = 0;
+
+        GF_ASSERT (frame);
+
+        if (!this || !data)
+                goto unwind;
+
+        args = data;
+        conf = this->private;
+
+        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
+                              remote_fd, op_errno, unwind);
+
+        req.fd = remote_fd;
+        req.offset = args->offset;
+        req.size = args->size;
+        memcpy(req.gfid, args->fd->inode->gfid, 16);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
+                                    req.xdata.xdata_len, op_errno, unwind);
+
+        ret = client_submit_request(this, &req, frame, conf->fops,
+                                    GFS3_OP_ZEROFILL, client3_3_zerofill_cbk,
+                                    NULL, NULL, 0, NULL, 0, NULL,
+                                    (xdrproc_t) xdr_gfs3_zerofill_req);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+unwind:
+        CLIENT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        GF_FREE (req.xdata.xdata_val);
+
+        return 0;
+}
 
 /* Table Specific to FOPS */
 
 
-rpc_clnt_procedure_t clnt3_1_fop_actors[GF_FOP_MAXVALUE] = {
+rpc_clnt_procedure_t clnt3_3_fop_actors[GF_FOP_MAXVALUE] = {
         [GF_FOP_NULL]        = { "NULL",        NULL},
-        [GF_FOP_STAT]        = { "STAT",        client3_1_stat },
-        [GF_FOP_READLINK]    = { "READLINK",    client3_1_readlink },
-        [GF_FOP_MKNOD]       = { "MKNOD",       client3_1_mknod },
-        [GF_FOP_MKDIR]       = { "MKDIR",       client3_1_mkdir },
-        [GF_FOP_UNLINK]      = { "UNLINK",      client3_1_unlink },
-        [GF_FOP_RMDIR]       = { "RMDIR",       client3_1_rmdir },
-        [GF_FOP_SYMLINK]     = { "SYMLINK",     client3_1_symlink },
-        [GF_FOP_RENAME]      = { "RENAME",      client3_1_rename },
-        [GF_FOP_LINK]        = { "LINK",        client3_1_link },
-        [GF_FOP_TRUNCATE]    = { "TRUNCATE",    client3_1_truncate },
-        [GF_FOP_OPEN]        = { "OPEN",        client3_1_open },
-        [GF_FOP_READ]        = { "READ",        client3_1_readv },
-        [GF_FOP_WRITE]       = { "WRITE",       client3_1_writev },
-        [GF_FOP_STATFS]      = { "STATFS",      client3_1_statfs },
-        [GF_FOP_FLUSH]       = { "FLUSH",       client3_1_flush },
-        [GF_FOP_FSYNC]       = { "FSYNC",       client3_1_fsync },
-        [GF_FOP_SETXATTR]    = { "SETXATTR",    client3_1_setxattr },
-        [GF_FOP_GETXATTR]    = { "GETXATTR",    client3_1_getxattr },
-        [GF_FOP_REMOVEXATTR] = { "REMOVEXATTR", client3_1_removexattr },
-        [GF_FOP_OPENDIR]     = { "OPENDIR",     client3_1_opendir },
-        [GF_FOP_FSYNCDIR]    = { "FSYNCDIR",    client3_1_fsyncdir },
-        [GF_FOP_ACCESS]      = { "ACCESS",      client3_1_access },
-        [GF_FOP_CREATE]      = { "CREATE",      client3_1_create },
-        [GF_FOP_FTRUNCATE]   = { "FTRUNCATE",   client3_1_ftruncate },
-        [GF_FOP_FSTAT]       = { "FSTAT",       client3_1_fstat },
-        [GF_FOP_LK]          = { "LK",          client3_1_lk },
-        [GF_FOP_LOOKUP]      = { "LOOKUP",      client3_1_lookup },
-        [GF_FOP_READDIR]     = { "READDIR",     client3_1_readdir },
-        [GF_FOP_INODELK]     = { "INODELK",     client3_1_inodelk },
-        [GF_FOP_FINODELK]    = { "FINODELK",    client3_1_finodelk },
-        [GF_FOP_ENTRYLK]     = { "ENTRYLK",     client3_1_entrylk },
-        [GF_FOP_FENTRYLK]    = { "FENTRYLK",    client3_1_fentrylk },
-        [GF_FOP_XATTROP]     = { "XATTROP",     client3_1_xattrop },
-        [GF_FOP_FXATTROP]    = { "FXATTROP",    client3_1_fxattrop },
-        [GF_FOP_FGETXATTR]   = { "FGETXATTR",   client3_1_fgetxattr },
-        [GF_FOP_FSETXATTR]   = { "FSETXATTR",   client3_1_fsetxattr },
-        [GF_FOP_RCHECKSUM]   = { "RCHECKSUM",   client3_1_rchecksum },
-        [GF_FOP_SETATTR]     = { "SETATTR",     client3_1_setattr },
-        [GF_FOP_FSETATTR]    = { "FSETATTR",    client3_1_fsetattr },
-        [GF_FOP_READDIRP]    = { "READDIRP",    client3_1_readdirp },
-        [GF_FOP_RELEASE]     = { "RELEASE",     client3_1_release },
-        [GF_FOP_RELEASEDIR]  = { "RELEASEDIR",  client3_1_releasedir },
+        [GF_FOP_STAT]        = { "STAT",        client3_3_stat },
+        [GF_FOP_READLINK]    = { "READLINK",    client3_3_readlink },
+        [GF_FOP_MKNOD]       = { "MKNOD",       client3_3_mknod },
+        [GF_FOP_MKDIR]       = { "MKDIR",       client3_3_mkdir },
+        [GF_FOP_UNLINK]      = { "UNLINK",      client3_3_unlink },
+        [GF_FOP_RMDIR]       = { "RMDIR",       client3_3_rmdir },
+        [GF_FOP_SYMLINK]     = { "SYMLINK",     client3_3_symlink },
+        [GF_FOP_RENAME]      = { "RENAME",      client3_3_rename },
+        [GF_FOP_LINK]        = { "LINK",        client3_3_link },
+        [GF_FOP_TRUNCATE]    = { "TRUNCATE",    client3_3_truncate },
+        [GF_FOP_OPEN]        = { "OPEN",        client3_3_open },
+        [GF_FOP_READ]        = { "READ",        client3_3_readv },
+        [GF_FOP_WRITE]       = { "WRITE",       client3_3_writev },
+        [GF_FOP_STATFS]      = { "STATFS",      client3_3_statfs },
+        [GF_FOP_FLUSH]       = { "FLUSH",       client3_3_flush },
+        [GF_FOP_FSYNC]       = { "FSYNC",       client3_3_fsync },
+        [GF_FOP_SETXATTR]    = { "SETXATTR",    client3_3_setxattr },
+        [GF_FOP_GETXATTR]    = { "GETXATTR",    client3_3_getxattr },
+        [GF_FOP_REMOVEXATTR] = { "REMOVEXATTR", client3_3_removexattr },
+        [GF_FOP_OPENDIR]     = { "OPENDIR",     client3_3_opendir },
+        [GF_FOP_FSYNCDIR]    = { "FSYNCDIR",    client3_3_fsyncdir },
+        [GF_FOP_ACCESS]      = { "ACCESS",      client3_3_access },
+        [GF_FOP_CREATE]      = { "CREATE",      client3_3_create },
+        [GF_FOP_FTRUNCATE]   = { "FTRUNCATE",   client3_3_ftruncate },
+        [GF_FOP_FSTAT]       = { "FSTAT",       client3_3_fstat },
+        [GF_FOP_LK]          = { "LK",          client3_3_lk },
+        [GF_FOP_LOOKUP]      = { "LOOKUP",      client3_3_lookup },
+        [GF_FOP_READDIR]     = { "READDIR",     client3_3_readdir },
+        [GF_FOP_INODELK]     = { "INODELK",     client3_3_inodelk },
+        [GF_FOP_FINODELK]    = { "FINODELK",    client3_3_finodelk },
+        [GF_FOP_ENTRYLK]     = { "ENTRYLK",     client3_3_entrylk },
+        [GF_FOP_FENTRYLK]    = { "FENTRYLK",    client3_3_fentrylk },
+        [GF_FOP_XATTROP]     = { "XATTROP",     client3_3_xattrop },
+        [GF_FOP_FXATTROP]    = { "FXATTROP",    client3_3_fxattrop },
+        [GF_FOP_FGETXATTR]   = { "FGETXATTR",   client3_3_fgetxattr },
+        [GF_FOP_FSETXATTR]   = { "FSETXATTR",   client3_3_fsetxattr },
+        [GF_FOP_RCHECKSUM]   = { "RCHECKSUM",   client3_3_rchecksum },
+        [GF_FOP_SETATTR]     = { "SETATTR",     client3_3_setattr },
+        [GF_FOP_FSETATTR]    = { "FSETATTR",    client3_3_fsetattr },
+        [GF_FOP_READDIRP]    = { "READDIRP",    client3_3_readdirp },
+	[GF_FOP_FALLOCATE]   = { "FALLOCATE",	client3_3_fallocate },
+	[GF_FOP_DISCARD]     = { "DISCARD",	client3_3_discard },
+        [GF_FOP_ZEROFILL]    = { "ZEROFILL",    client3_3_zerofill},
+        [GF_FOP_RELEASE]     = { "RELEASE",     client3_3_release },
+        [GF_FOP_RELEASEDIR]  = { "RELEASEDIR",  client3_3_releasedir },
         [GF_FOP_GETSPEC]     = { "GETSPEC",     client3_getspec },
-        [GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_1_fremovexattr },
+        [GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_3_fremovexattr },
 };
 
 /* Used From RPC-CLNT library to log proper name of procedure based on number */
-char *clnt3_1_fop_names[GFS3_OP_MAXVALUE] = {
+char *clnt3_3_fop_names[GFS3_OP_MAXVALUE] = {
         [GFS3_OP_NULL]        = "NULL",
         [GFS3_OP_STAT]        = "STAT",
         [GFS3_OP_READLINK]    = "READLINK",
@@ -5288,13 +6193,17 @@ char *clnt3_1_fop_names[GFS3_OP_MAXVALUE] = {
         [GFS3_OP_RELEASE]     = "RELEASE",
         [GFS3_OP_RELEASEDIR]  = "RELEASEDIR",
         [GFS3_OP_FREMOVEXATTR] = "FREMOVEXATTR",
+	[GFS3_OP_FALLOCATE]   = "FALLOCATE",
+	[GFS3_OP_DISCARD]     = "DISCARD",
+        [GFS3_OP_ZEROFILL]    = "ZEROFILL",
+
 };
 
-rpc_clnt_prog_t clnt3_1_fop_prog = {
-        .progname  = "GlusterFS 3.1",
-        .prognum   = GLUSTER3_1_FOP_PROGRAM,
-        .progver   = GLUSTER3_1_FOP_VERSION,
-        .numproc   = GLUSTER3_1_FOP_PROCCNT,
-        .proctable = clnt3_1_fop_actors,
-        .procnames = clnt3_1_fop_names,
+rpc_clnt_prog_t clnt3_3_fop_prog = {
+        .progname  = "GlusterFS 3.3",
+        .prognum   = GLUSTER_FOP_PROGRAM,
+        .progver   = GLUSTER_FOP_VERSION,
+        .numproc   = GLUSTER_FOP_PROCCNT,
+        .proctable = clnt3_3_fop_actors,
+        .procnames = clnt3_3_fop_names,
 };
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 3567dedbd..306e555ef 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -116,22 +107,58 @@ out:
         return;
 }
 
+int32_t
+client_register_grace_timer (xlator_t *this, clnt_conf_t *conf)
+{
+        int32_t  ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("client", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        pthread_mutex_lock (&conf->lock);
+        {
+                if (conf->grace_timer || !conf->grace_timer_needed) {
+                        gf_log (this->name, GF_LOG_TRACE,
+				"Client grace timer is already set "
+				"or a grace-timer has already time out, "
+				"not registering a new timer");
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "Registering a grace timer");
+
+                        conf->grace_timer_needed = _gf_false;
+
+                        conf->grace_timer =
+                                gf_timer_call_after (this->ctx,
+                                                     conf->grace_ts,
+                                                     client_grace_timeout,
+                                                     conf->rpc);
+                }
+        }
+        pthread_mutex_unlock (&conf->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
 int
 client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
-                       rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbk,
+                       rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
                        struct iobref *iobref,  struct iovec *rsphdr,
                        int rsphdr_count, struct iovec *rsp_payload,
                        int rsp_payload_count, struct iobref *rsp_iobref,
                        xdrproc_t xdrproc)
 {
-        int            ret         = -1;
-        clnt_conf_t   *conf        = NULL;
-        struct iovec   iov         = {0, };
-        struct iobuf  *iobuf       = NULL;
-        int            count       = 0;
-        char           start_ping  = 0;
-        struct iobref *new_iobref  = NULL;
-        ssize_t        xdr_size    = 0;
+        int             ret        = -1;
+        clnt_conf_t    *conf       = NULL;
+        struct iovec    iov        = {0, };
+        struct iobuf   *iobuf      = NULL;
+        int             count      = 0;
+        char            start_ping = 0;
+        struct iobref  *new_iobref = NULL;
+        ssize_t         xdr_size   = 0;
+        struct rpc_req  rpcreq     = {0, };
 
         GF_VALIDATE_OR_GOTO ("client", this, out);
         GF_VALIDATE_OR_GOTO (this->name, prog, out);
@@ -199,8 +226,8 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
         }
 
         /* Send the msg */
-        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count, NULL,
-                               0, new_iobref, frame, rsphdr, rsphdr_count,
+        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbkfn, &iov, count,
+                               NULL, 0, new_iobref, frame, rsphdr, rsphdr_count,
                                rsp_payload, rsp_payload_count, rsp_iobref);
 
         if (ret < 0) {
@@ -221,14 +248,27 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
                 client_start_ping ((void *) this);
 
         ret = 0;
-out:
-        if (new_iobref != NULL)
+
+        if (new_iobref)
                 iobref_unref (new_iobref);
 
         if (iobuf)
                 iobuf_unref (iobuf);
 
         return ret;
+
+out:
+        rpcreq.rpc_status = -1;
+
+        cbkfn (&rpcreq, NULL, 0, frame);
+
+        if (new_iobref)
+                iobref_unref (new_iobref);
+
+        if (iobuf)
+                iobuf_unref (iobuf);
+
+        return 0;
 }
 
 
@@ -246,7 +286,6 @@ client_releasedir (xlator_t *this, fd_t *fd)
         clnt_conf_t *conf = NULL;
         rpc_clnt_procedure_t *proc = NULL;
         clnt_args_t  args = {0,};
-        call_frame_t *frame = NULL;
 
         conf = this->private;
         if (!conf || !conf->fops)
@@ -262,11 +301,7 @@ client_releasedir (xlator_t *this, fd_t *fd)
                 goto out;
         }
         if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                ret = proc->fn (frame, this, &args);
+                ret = proc->fn (NULL, this, &args);
         }
 out:
         if (ret)
@@ -282,7 +317,6 @@ client_release (xlator_t *this, fd_t *fd)
         clnt_conf_t *conf = NULL;
         rpc_clnt_procedure_t *proc = NULL;
         clnt_args_t  args = {0,};
-        call_frame_t *frame = NULL;
 
         conf = this->private;
         if (!conf || !conf->fops)
@@ -297,11 +331,7 @@ client_release (xlator_t *this, fd_t *fd)
                 goto out;
         }
         if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                ret = proc->fn (frame, this, &args);
+                ret = proc->fn (NULL, this, &args);
         }
 out:
         if (ret)
@@ -313,7 +343,7 @@ out:
 
 int32_t
 client_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xattr_req)
+               dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -325,7 +355,7 @@ client_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
 
         args.loc = loc;
-        args.dict = xattr_req;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_LOOKUP];
         if (!proc) {
@@ -347,7 +377,7 @@ out:
 
 
 int32_t
-client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -359,6 +389,7 @@ client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
 
         args.loc = loc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_STAT];
         if (!proc) {
@@ -371,14 +402,15 @@ client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (stat, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (stat, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
 
 
 int32_t
-client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 off_t offset, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -391,6 +423,7 @@ client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
 
         args.loc    = loc;
         args.offset = offset;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_TRUNCATE];
         if (!proc) {
@@ -403,7 +436,7 @@ client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (truncate, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (truncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 
 	return 0;
@@ -411,7 +444,8 @@ out:
 
 
 int32_t
-client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  off_t offset, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -424,6 +458,7 @@ client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
 
         args.fd     = fd;
         args.offset = offset;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FTRUNCATE];
         if (!proc) {
@@ -436,7 +471,7 @@ client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -444,7 +479,8 @@ out:
 
 
 int32_t
-client_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+client_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int32_t mask, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -457,6 +493,7 @@ client_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
 
         args.loc  = loc;
         args.mask = mask;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_ACCESS];
         if (!proc) {
@@ -469,7 +506,7 @@ client_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (access, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (access, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -478,7 +515,8 @@ out:
 
 
 int32_t
-client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 size_t size, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -491,6 +529,7 @@ client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
 
         args.loc  = loc;
         args.size = size;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_READLINK];
         if (!proc) {
@@ -503,7 +542,7 @@ client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (readlink, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (readlink, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -511,7 +550,7 @@ out:
 
 int
 client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, dict_t *params)
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -525,7 +564,8 @@ client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         args.loc  = loc;
         args.mode = mode;
         args.rdev = rdev;
-        args.dict = params;
+        args.umask = umask;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_MKNOD];
         if (!proc) {
@@ -539,7 +579,7 @@ client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (mknod, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -547,7 +587,7 @@ out:
 
 int
 client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              mode_t mode, dict_t *params)
+              mode_t mode, mode_t umask, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -560,7 +600,8 @@ client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         args.loc  = loc;
         args.mode = mode;
-        args.dict = params;
+        args.umask = umask;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_MKDIR];
         if (!proc) {
@@ -574,7 +615,7 @@ client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (mkdir, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -582,7 +623,8 @@ out:
 
 
 int32_t
-client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int xflag, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -594,6 +636,8 @@ client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
 
         args.loc = loc;
+        args.xdata = xdata;
+        args.flags = xflag;
 
         proc = &conf->fops->proctable[GF_FOP_UNLINK];
         if (!proc) {
@@ -607,13 +651,14 @@ client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 out:
         if (ret)
                 STACK_UNWIND_STRICT (unlink, frame, -1, ENOTCONN,
-                                     NULL, NULL);
+                                     NULL, NULL, NULL);
 
 	return 0;
 }
 
 int32_t
-client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -626,6 +671,7 @@ client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 
         args.loc = loc;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_RMDIR];
         if (!proc) {
@@ -640,7 +686,7 @@ out:
         /* think of avoiding a missing frame */
         if (ret)
                 STACK_UNWIND_STRICT (rmdir, frame, -1, ENOTCONN,
-                                     NULL, NULL);
+                                     NULL, NULL, NULL);
 
 	return 0;
 }
@@ -648,7 +694,7 @@ out:
 
 int
 client_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                loc_t *loc, dict_t *params)
+                loc_t *loc, mode_t umask, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -661,7 +707,8 @@ client_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
 
         args.linkname = linkpath;
         args.loc      = loc;
-        args.dict     = params;
+        args.umask    = umask;
+        args.xdata    = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_SYMLINK];
         if (!proc) {
@@ -675,7 +722,7 @@ client_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (symlink, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -684,7 +731,7 @@ out:
 
 int32_t
 client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
+               loc_t *newloc, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -697,6 +744,8 @@ client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 
         args.oldloc = oldloc;
         args.newloc = newloc;
+        args.xdata  = xdata;
+
         proc = &conf->fops->proctable[GF_FOP_RENAME];
         if (!proc) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -709,7 +758,7 @@ client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (rename, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -718,7 +767,7 @@ out:
 
 int32_t
 client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-             loc_t *newloc)
+             loc_t *newloc, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -731,6 +780,7 @@ client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 
         args.oldloc = oldloc;
         args.newloc = newloc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_LINK];
         if (!proc) {
@@ -744,7 +794,7 @@ client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (link, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -752,8 +802,8 @@ out:
 
 
 int32_t
-client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,  int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -765,10 +815,15 @@ client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
 
         args.loc = loc;
-        args.flags = flags;
         args.mode = mode;
         args.fd = fd;
-        args.dict = params;
+        args.umask = umask;
+        args.xdata = xdata;
+
+        if (!conf->filter_o_direct)
+                args.flags = flags;
+        else
+                args.flags = (flags & ~O_DIRECT);
 
         proc = &conf->fops->proctable[GF_FOP_CREATE];
         if (!proc) {
@@ -782,7 +837,7 @@ client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (create, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL);
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -791,7 +846,7 @@ out:
 
 int32_t
 client_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, int32_t wbflags)
+             int32_t flags, fd_t *fd, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -803,9 +858,13 @@ client_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
 
         args.loc = loc;
-        args.flags = flags;
         args.fd = fd;
-        args.wbflags = wbflags;
+        args.xdata = xdata;
+
+        if (!conf->filter_o_direct)
+                args.flags = flags;
+        else
+                args.flags = (flags & ~O_DIRECT);
 
         proc = &conf->fops->proctable[GF_FOP_OPEN];
         if (!proc) {
@@ -819,7 +878,7 @@ client_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 out:
         if (ret)
-                STACK_UNWIND_STRICT (open, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (open, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -828,7 +887,7 @@ out:
 
 int32_t
 client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, uint32_t flags)
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -843,6 +902,7 @@ client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         args.size   = size;
         args.offset = offset;
         args.flags  = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_READ];
         if (!proc) {
@@ -857,7 +917,7 @@ client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 out:
         if (ret)
                 STACK_UNWIND_STRICT (readv, frame, -1, ENOTCONN,
-                                     NULL, 0, NULL, NULL);
+                                     NULL, 0, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -868,7 +928,7 @@ out:
 int32_t
 client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                struct iovec *vector, int32_t count, off_t off,
-               uint32_t flags, struct iobref *iobref)
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -883,8 +943,10 @@ client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         args.vector = vector;
         args.count  = count;
         args.offset = off;
+        args.size   = iov_length (vector, count);
         args.flags  = flags;
         args.iobref = iobref;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_WRITE];
         if (!proc) {
@@ -897,14 +959,14 @@ client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
 
 
 int32_t
-client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -916,6 +978,7 @@ client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto out;
 
         args.fd = fd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FLUSH];
         if (!proc) {
@@ -928,7 +991,7 @@ client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (flush, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (flush, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -937,7 +1000,7 @@ out:
 
 int32_t
 client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t flags)
+              int32_t flags, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -950,6 +1013,7 @@ client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         args.fd    = fd;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSYNC];
         if (!proc) {
@@ -962,7 +1026,7 @@ client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fsync, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (fsync, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
@@ -970,7 +1034,7 @@ out:
 
 
 int32_t
-client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -982,6 +1046,7 @@ client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto out;
 
         args.fd = fd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSTAT];
         if (!proc) {
@@ -994,7 +1059,7 @@ client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fstat, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (fstat, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1002,7 +1067,8 @@ out:
 
 
 int32_t
-client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1015,6 +1081,7 @@ client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 
         args.loc = loc;
         args.fd  = fd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_OPENDIR];
         if (!proc) {
@@ -1027,7 +1094,7 @@ client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (opendir, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (opendir, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1035,7 +1102,7 @@ out:
 
 
 int32_t
-client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1048,6 +1115,7 @@ client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
 
         args.fd    = fd;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSYNCDIR];
         if (!proc) {
@@ -1060,7 +1128,7 @@ client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1068,7 +1136,7 @@ out:
 
 
 int32_t
-client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1080,6 +1148,7 @@ client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 goto out;
 
         args.loc = loc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_STATFS];
         if (!proc) {
@@ -1092,7 +1161,7 @@ client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (statfs, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (statfs, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1209,8 +1278,7 @@ client_set_remote_options (char *value, xlator_t *this)
 
         ret = _gf_true;
 out:
-        if (dup_value)
-                GF_FREE (dup_value);
+        GF_FREE (dup_value);
 
         return ret;
 }
@@ -1218,7 +1286,7 @@ out:
 
 int32_t
 client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                 int32_t flags)
+                 int32_t flags, dict_t *xdata)
 {
         int                   ret         = -1;
         int                   op_ret      = -1;
@@ -1266,8 +1334,9 @@ client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
         }
 
         args.loc   = loc;
-        args.dict  = dict;
+        args.xattr = dict;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_SETXATTR];
         if (!proc) {
@@ -1284,7 +1353,7 @@ client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
         }
 out:
         if (need_unwind)
-                STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+                STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
 
 	return 0;
 }
@@ -1293,7 +1362,7 @@ out:
 
 int32_t
 client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int32_t flags)
+                  dict_t *dict, int32_t flags, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1304,9 +1373,10 @@ client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (!conf || !conf->fops)
                 goto out;
 
-        args.fd = fd;
-        args.dict = dict;
+        args.fd    = fd;
+        args.xattr = dict;
         args.flags = flags;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSETXATTR];
         if (!proc) {
@@ -1319,7 +1389,7 @@ client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1329,7 +1399,7 @@ out:
 
 int32_t
 client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1342,6 +1412,7 @@ client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         args.fd = fd;
         args.name = name;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FGETXATTR];
         if (!proc) {
@@ -1354,7 +1425,7 @@ client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1363,7 +1434,7 @@ out:
 
 int32_t
 client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name)
+                 const char *name, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1376,6 +1447,7 @@ client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         args.name = name;
         args.loc  = loc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_GETXATTR];
         if (!proc) {
@@ -1388,7 +1460,7 @@ client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1397,7 +1469,7 @@ out:
 
 int32_t
 client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                gf_xattrop_flags_t flags, dict_t *dict)
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1410,7 +1482,8 @@ client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         args.loc = loc;
         args.flags = flags;
-        args.dict = dict;
+        args.xattr = dict;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_XATTROP];
         if (!proc) {
@@ -1423,7 +1496,7 @@ client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1432,7 +1505,7 @@ out:
 
 int32_t
 client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 gf_xattrop_flags_t flags, dict_t *dict)
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1445,7 +1518,8 @@ client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         args.fd = fd;
         args.flags = flags;
-        args.dict = dict;
+        args.xattr = dict;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FXATTROP];
         if (!proc) {
@@ -1458,7 +1532,7 @@ client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1467,7 +1541,7 @@ out:
 
 int32_t
 client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name)
+                    const char *name, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1480,6 +1554,7 @@ client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         args.name = name;
         args.loc  = loc;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_REMOVEXATTR];
         if (!proc) {
@@ -1492,14 +1567,14 @@ client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
 
 int32_t
 client_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name)
+                     const char *name, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1512,6 +1587,7 @@ client_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         args.name = name;
         args.fd  = fd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FREMOVEXATTR];
         if (!proc) {
@@ -1524,14 +1600,14 @@ client_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
 
 int32_t
 client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock)
+           struct gf_flock *lock, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1545,6 +1621,7 @@ client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         args.fd    = fd;
         args.cmd   = cmd;
         args.flock = lock;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_LK];
         if (!proc) {
@@ -1557,7 +1634,7 @@ client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (lk, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (lk, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1565,7 +1642,7 @@ out:
 
 int32_t
 client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                loc_t *loc, int32_t cmd, struct gf_flock *lock)
+                loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1580,6 +1657,7 @@ client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
         args.cmd    = cmd;
         args.flock  = lock;
         args.volume = volume;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_INODELK];
         if (!proc) {
@@ -1592,7 +1670,7 @@ client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1601,7 +1679,7 @@ out:
 
 int32_t
 client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                 fd_t *fd, int32_t cmd, struct gf_flock *lock)
+                 fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1616,6 +1694,7 @@ client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
         args.cmd    = cmd;
         args.flock  = lock;
         args.volume = volume;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FINODELK];
         if (!proc) {
@@ -1628,7 +1707,7 @@ client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1637,7 +1716,7 @@ out:
 int32_t
 client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 loc_t *loc, const char *basename, entrylk_cmd cmd,
-                entrylk_type type)
+                entrylk_type type, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1653,6 +1732,7 @@ client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
         args.type         = type;
         args.volume       = volume;
         args.cmd_entrylk  = cmd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_ENTRYLK];
         if (!proc) {
@@ -1665,7 +1745,7 @@ client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1675,7 +1755,7 @@ out:
 int32_t
 client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                  fd_t *fd, const char *basename, entrylk_cmd cmd,
-                 entrylk_type type)
+                 entrylk_type type, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1691,6 +1771,7 @@ client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
         args.type         = type;
         args.volume       = volume;
         args.cmd_entrylk  = cmd;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FENTRYLK];
         if (!proc) {
@@ -1703,7 +1784,7 @@ client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOTCONN);
+                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOTCONN, NULL);
 
 	return 0;
 }
@@ -1711,7 +1792,7 @@ out:
 
 int32_t
 client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                  int32_t len)
+                  int32_t len, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1725,6 +1806,7 @@ client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         args.fd = fd;
         args.offset = offset;
         args.len = len;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_RCHECKSUM];
         if (!proc) {
@@ -1737,14 +1819,14 @@ client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOTCONN, 0, NULL);
+                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOTCONN, 0, NULL, NULL);
 
 	return 0;
 }
 
 int32_t
 client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                size_t size, off_t off)
+                size_t size, off_t off, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1758,6 +1840,7 @@ client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
         args.fd = fd;
         args.size = size;
         args.offset = off;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_READDIR];
         if (!proc) {
@@ -1770,7 +1853,7 @@ client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1792,7 +1875,7 @@ client_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
         args.fd = fd;
         args.size = size;
         args.offset = off;
-        args.dict = dict;
+        args.xdata = dict;
 
         proc = &conf->fops->proctable[GF_FOP_READDIRP];
         if (!proc) {
@@ -1805,7 +1888,7 @@ client_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOTCONN, NULL);
+                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOTCONN, NULL, NULL);
 
 	return 0;
 }
@@ -1813,7 +1896,7 @@ out:
 
 int32_t
 client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1827,6 +1910,7 @@ client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         args.loc = loc;
         args.stbuf = stbuf;
         args.valid = valid;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_SETATTR];
         if (!proc) {
@@ -1839,14 +1923,14 @@ client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (setattr, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (setattr, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
 
 int32_t
 client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int          ret  = -1;
         clnt_conf_t *conf = NULL;
@@ -1860,6 +1944,7 @@ client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         args.fd = fd;
         args.stbuf = stbuf;
         args.valid = valid;
+        args.xdata = xdata;
 
         proc = &conf->fops->proctable[GF_FOP_FSETATTR];
         if (!proc) {
@@ -1872,11 +1957,115 @@ client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 ret = proc->fn (frame, this, &args);
 out:
         if (ret)
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOTCONN, NULL, NULL);
+                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
 	return 0;
 }
 
+int32_t
+client_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        int          ret  = -1;
+        clnt_conf_t *conf = NULL;
+        rpc_clnt_procedure_t *proc = NULL;
+        clnt_args_t  args = {0,};
+
+        conf = this->private;
+        if (!conf || !conf->fops)
+                goto out;
+
+        args.fd = fd;
+	args.flags = mode;
+	args.offset = offset;
+	args.size = len;
+        args.xdata = xdata;
+
+        proc = &conf->fops->proctable[GF_FOP_FALLOCATE];
+        if (!proc) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpc procedure not found for %s",
+                        gf_fop_list[GF_FOP_FALLOCATE]);
+                goto out;
+        }
+        if (proc->fn)
+                ret = proc->fn (frame, this, &args);
+out:
+        if (ret)
+                STACK_UNWIND_STRICT (fallocate, frame, -1, ENOTCONN, NULL, NULL, NULL);
+
+	return 0;
+}
+
+int32_t
+client_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        int          ret  = -1;
+        clnt_conf_t *conf = NULL;
+        rpc_clnt_procedure_t *proc = NULL;
+        clnt_args_t  args = {0,};
+
+        conf = this->private;
+        if (!conf || !conf->fops)
+                goto out;
+
+        args.fd = fd;
+	args.offset = offset;
+	args.size = len;
+        args.xdata = xdata;
+
+        proc = &conf->fops->proctable[GF_FOP_DISCARD];
+        if (!proc) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpc procedure not found for %s",
+                        gf_fop_list[GF_FOP_DISCARD]);
+                goto out;
+        }
+        if (proc->fn)
+                ret = proc->fn (frame, this, &args);
+out:
+        if (ret)
+                STACK_UNWIND_STRICT(discard, frame, -1, ENOTCONN, NULL, NULL, NULL);
+
+	return 0;
+}
+
+int32_t
+client_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
+{
+        int          ret              = -1;
+        clnt_conf_t *conf             = NULL;
+        rpc_clnt_procedure_t *proc    = NULL;
+        clnt_args_t  args             = {0,};
+
+        conf = this->private;
+        if (!conf || !conf->fops)
+                goto out;
+
+        args.fd = fd;
+        args.offset = offset;
+        args.size = len;
+        args.xdata = xdata;
+
+        proc = &conf->fops->proctable[GF_FOP_ZEROFILL];
+        if (!proc) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpc procedure not found for %s",
+                        gf_fop_list[GF_FOP_ZEROFILL]);
+                goto out;
+        }
+        if (proc->fn)
+                ret = proc->fn (frame, this, &args);
+out:
+        if (ret)
+                STACK_UNWIND_STRICT(zerofill, frame, -1, ENOTCONN,
+                                    NULL, NULL, NULL);
+
+        return 0;
+}
+
 
 int32_t
 client_getspec (call_frame_t *frame, xlator_t *this, const char *key,
@@ -1966,7 +2155,7 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
 
                 if ((ret < 0) || (strcasecmp (handshake, "on"))) {
-                        ret = client_handshake (this, conf->rpc);
+                        ret = client_handshake (this, rpc);
                         if (ret)
                                 gf_log (this->name, GF_LOG_WARNING,
                                         "handshake msg returned %d", ret);
@@ -1985,7 +2174,7 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                 /* Cancel grace timer if set */
                 pthread_mutex_lock (&conf->lock);
                 {
-                        conf->grace_timer_flag = _gf_true;
+                        conf->grace_timer_needed = _gf_true;
 
                         if (conf->grace_timer) {
                                 gf_log (this->name, GF_LOG_WARNING,
@@ -2002,33 +2191,25 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                 break;
         }
         case RPC_CLNT_DISCONNECT:
-                /* client_mark_fd_bad (this); */
-
-                pthread_mutex_lock (&conf->lock);
-                {
-                        if (conf->grace_timer || !conf->grace_timer_flag) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "Client grace timer is already set "
-                                        "or a grace-timer has already timeout, "
-                                        "not registering a new timer");
-                        } else {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Registering a grace timer");
-                                conf->grace_timer_flag = _gf_false;
-
-                                conf->grace_timer =
-                                        gf_timer_call_after (this->ctx,
-                                                             conf->grace_tv,
-                                                             client_grace_timeout,
-                                                             conf->rpc);
-                        }
-                }
-                pthread_mutex_unlock (&conf->lock);
+                if (!conf->lk_heal)
+                        client_mark_fd_bad (this);
+                else
+                        client_register_grace_timer (this, conf);
 
                 if (!conf->skip_notify) {
-                        if (conf->connected)
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "disconnected");
+                        if (conf->connected) {
+                               gf_log (this->name,
+                                        ((!conf->disconnect_err_logged)
+                                        ? GF_LOG_INFO : GF_LOG_DEBUG),
+                                        "disconnected from %s. Client process "
+                                        "will keep trying to connect to "
+                                        "glusterd until brick's port is "
+                                        "available",
+                                  conf->rpc->conn.name);
+
+                                if (conf->portmap_err_logged)
+                                        conf->disconnect_err_logged = 1;
+                        }
 
                         /* If the CHILD_DOWN event goes to parent xlator
                            multiple times, the logic of parent xlator notify
@@ -2052,6 +2233,15 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                 conf->connected = 0;
                 conf->skip_notify = 0;
 
+                if (conf->quick_reconnect) {
+                        conf->quick_reconnect = 0;
+                        rpc_clnt_start (rpc);
+
+                } else {
+                        rpc->conn.config.remote_port = 0;
+
+                }
+
                 break;
 
         default:
@@ -2135,6 +2325,9 @@ build_client_config (xlator_t *this, clnt_conf_t *conf)
                 gf_log (this->name, GF_LOG_WARNING,
                         "option 'remote-subvolume' not given");
 
+        GF_OPTION_INIT ("filter-O_DIRECT", conf->filter_o_direct,
+                        bool, out);
+
         ret = 0;
 out:
         return ret;
@@ -2171,6 +2364,9 @@ client_destroy_rpc (xlator_t *this)
                 goto out;
 
         if (conf->rpc) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&conf->rpc->conn);
+
                 conf->rpc = rpc_clnt_unref (conf->rpc);
                 ret = 0;
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -2216,7 +2412,8 @@ client_init_rpc (xlator_t *this)
         conf->handshake = &clnt_handshake_prog;
         conf->dump      = &clnt_dump_prog;
 
-        ret = rpcclnt_cbk_program_register (conf->rpc, &gluster_cbk_prog);
+        ret = rpcclnt_cbk_program_register (conf->rpc, &gluster_cbk_prog,
+                                            this);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to register callback program");
@@ -2243,22 +2440,25 @@ client_init_grace_timer (xlator_t *this, dict_t *options,
         GF_VALIDATE_OR_GOTO (this->name, options, out);
         GF_VALIDATE_OR_GOTO (this->name, conf, out);
 
-        conf->lk_heal = _gf_true;
+        conf->lk_heal = _gf_false;
 
         ret = dict_get_str (options, "lk-heal", &lk_heal);
         if (!ret)
                 gf_string2boolean (lk_heal, &conf->lk_heal);
 
+        gf_log (this->name, GF_LOG_DEBUG, "lk-heal = %s",
+                (conf->lk_heal) ? "on" : "off");
+
         ret = dict_get_int32 (options, "grace-timeout", &grace_timeout);
         if (!ret)
-                conf->grace_tv.tv_sec = grace_timeout;
+                conf->grace_ts.tv_sec = grace_timeout;
         else
-                conf->grace_tv.tv_sec = 10;
+                conf->grace_ts.tv_sec = 10;
 
-        conf->grace_tv.tv_usec  = 0;
+        conf->grace_ts.tv_nsec  = 0;
 
-        gf_log (this->name, GF_LOG_INFO, "lk-heal = %s",
-                (conf->lk_heal) ? "on" : "off");
+        gf_log (this->name, GF_LOG_DEBUG, "Client grace timeout "
+                "value = %"PRIu64, conf->grace_ts.tv_sec);
 
         ret = 0;
 out:
@@ -2312,6 +2512,9 @@ reconfigure (xlator_t *this, dict_t *options)
                 }
         }
 
+        GF_OPTION_RECONF ("filter-O_DIRECT", conf->filter_o_direct,
+                          options, bool, out);
+
         ret = client_init_grace_timer (this, options, conf);
         if (ret)
                 goto out;
@@ -2349,9 +2552,9 @@ init (xlator_t *this)
         INIT_LIST_HEAD (&conf->saved_fds);
 
         /* Initialize parameters for lock self healing*/
-        conf->lk_version       = 1;
-        conf->grace_timer      = NULL;
-        conf->grace_timer_flag = _gf_true;
+        conf->lk_version         = 1;
+        conf->grace_timer        = NULL;
+        conf->grace_timer_needed = _gf_true;
 
         ret = client_init_grace_timer (this, this->options, conf);
         if (ret)
@@ -2403,8 +2606,12 @@ fini (xlator_t *this)
         this->private = NULL;
 
         if (conf) {
-                if (conf->rpc)
-                       rpc_clnt_unref (conf->rpc);
+                if (conf->rpc) {
+                        /* cleanup the saved-frames before last unref */
+                        rpc_clnt_connection_cleanup (&conf->rpc->conn);
+
+                        rpc_clnt_unref (conf->rpc);
+                }
 
                 /* Saved Fds */
                 /* TODO: */
@@ -2498,8 +2705,6 @@ client_priv_dump (xlator_t *this)
         }
 
         gf_proc_dump_write("connecting", "%d", conf->connecting);
-        gf_proc_dump_write("last_sent", "%s", ctime(&conf->last_sent.tv_sec));
-        gf_proc_dump_write("last_received", "%s", ctime(&conf->last_received.tv_sec));
 
         if (conf->rpc) {
                 gf_proc_dump_write("total_bytes_read", "%"PRIu64,
@@ -2579,6 +2784,9 @@ struct xlator_fops fops = {
         .fxattrop    = client_fxattrop,
         .setattr     = client_setattr,
         .fsetattr    = client_fsetattr,
+	.fallocate   = client_fallocate,
+	.discard     = client_discard,
+        .zerofill    = client_zerofill,
         .getspec     = client_getspec,
 };
 
@@ -2632,15 +2840,35 @@ struct volume_options options[] = {
           .type  = GF_OPTION_TYPE_BOOL
         },
         { .key   = {"lk-heal"},
-          .type  = GF_OPTION_TYPE_STR
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "When the connection to client is lost, server "
+                         "cleans up all the locks held by the client. After "
+                         "the connection is restored, the client reacquires "
+                         "(heals) the fcntl locks released by the server."
         },
         { .key   = {"grace-timeout"},
-          .type  = GF_OPTION_TYPE_INT
+          .type  = GF_OPTION_TYPE_INT,
+          .min   = 10,
+          .max   = 1800,
+          .default_value = "10",
+          .description = "Specifies the duration for the lock state to be "
+                         "maintained on the client after a network "
+                         "disconnection. Range 10-1800 seconds."
         },
         {.key  = {"tcp-window-size"},
          .type = GF_OPTION_TYPE_SIZET,
          .min  = GF_MIN_SOCKET_WINDOW_SIZE,
-         .max  = GF_MAX_SOCKET_WINDOW_SIZE
+         .max  = GF_MAX_SOCKET_WINDOW_SIZE,
+         .description = "Specifies the window size for tcp socket."
+        },
+        { .key   = {"filter-O_DIRECT"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "disable",
+          .description = "If enabled, in open() and creat() calls, O_DIRECT "
+          "flag will be filtered at the client protocol level so server will "
+          "still continue to cache the file. This works similar to NFS's "
+          "behavior of O_DIRECT",
         },
         { .key   = {NULL} },
 };
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 7203556f5..bc0f5d0e9 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CLIENT_H
@@ -38,27 +29,40 @@
 #define GF_MAX_SOCKET_WINDOW_SIZE  (1 * GF_UNIT_MB)
 #define GF_MIN_SOCKET_WINDOW_SIZE  (0)
 
-#define CLIENT_GET_REMOTE_FD(conf, fd, remote_fd, op_errno, label)      \
+typedef enum {
+        GF_LK_HEAL_IN_PROGRESS,
+        GF_LK_HEAL_DONE,
+} lk_heal_state_t;
+
+typedef enum {
+        DEFAULT_REMOTE_FD = 0,
+        FALLBACK_TO_ANON_FD = 1
+} clnt_remote_fd_flags_t;
+
+#define CLIENT_GET_REMOTE_FD(xl, fd, flags, remote_fd, op_errno, label) \
         do {                                                            \
-                clnt_fd_ctx_t      *fdctx    = NULL;                    \
-                pthread_mutex_lock (&conf->lock);                       \
-                {                                                       \
-                        fdctx = this_fd_get_ctx (fd, THIS);             \
-                }                                                       \
-                pthread_mutex_unlock (&conf->lock);                     \
-                if (!fdctx) {                                           \
-                        remote_fd = -2;                                 \
-                } else {                                                \
-                        remote_fd = fdctx->remote_fd;                   \
+                int     _ret    = 0;                                    \
+                _ret = client_get_remote_fd (xl, fd, flags, &remote_fd);\
+                if (_ret < 0) {                                         \
+                        op_errno = errno;                               \
+                        goto label;                                     \
                 }                                                       \
                 if (remote_fd == -1) {                                  \
-                        gf_log (THIS->name, GF_LOG_WARNING, " (%s) "    \
+                        gf_log (xl->name, GF_LOG_WARNING, " (%s) "      \
                                 "remote_fd is -1. EBADFD",              \
                                 uuid_utoa (fd->inode->gfid));           \
                         op_errno = EBADFD;                              \
                         goto label;                                     \
                 }                                                       \
-        } while (0);
+        } while (0)
+
+#define CLIENT_STACK_UNWIND(op, frame, params ...) do {             \
+                clnt_local_t *__local = frame->local;               \
+                frame->local = NULL;                                \
+                STACK_UNWIND_STRICT (op, frame, params);            \
+                client_local_wipe (__local);                        \
+        } while (0)
+
 
 struct clnt_options {
         char *remote_subvolume;
@@ -73,8 +77,6 @@ typedef struct clnt_conf {
         pthread_mutex_t        lock;
         int                    connecting;
         int                    connected;
-	struct timeval         last_sent;
-	struct timeval         last_received;
 
         rpc_clnt_prog_t       *fops;
         rpc_clnt_prog_t       *mgmt;
@@ -91,6 +93,10 @@ typedef struct clnt_conf {
                                                    which was sent earlier */
         char                   portmap_err_logged; /* flag used to prevent
                                                       excessive logging */
+        char                   disconnect_err_logged; /* flag used to prevent
+                                                         excessive disconnect
+                                                         logging */
+
         char                   need_different_port; /* flag used to change the
                                                        portmap path in case of
                                                        'tcp,rdma' on server */
@@ -98,15 +104,27 @@ typedef struct clnt_conf {
         uint16_t               lk_version; /* this variable is used to distinguish
                                               client-server transaction while
                                               performing lock healing */
-        struct timeval         grace_tv;
+        struct timespec        grace_ts;
         gf_timer_t            *grace_timer;
-        gf_boolean_t           grace_timer_flag; /* The state of this flag will
-                                                    be used to decide whether
-                                                    a new grace-timer must be
-                                                    registered or not. False
-                                                    means dont register, true
-                                                    means register */
+        gf_boolean_t           grace_timer_needed; /* The state of this flag will
+                                                      be used to decide whether
+                                                      a new grace-timer must be
+                                                      registered or not. False
+                                                      means dont register, true
+                                                      means register */
         char                   parent_down;
+	gf_boolean_t           quick_reconnect; /* When reconnecting after
+						   portmap query, do not let
+						   the reconnection happen after
+						   the usual 3-second wait
+						*/
+        gf_boolean_t           filter_o_direct; /* if set, filter O_DIRECT from
+                                                   the flags list of open() */
+        /* set volume is the op which results in creating/re-using
+         * the conn-id and is called once per connection, this remembers
+         * how manytimes set_volume is called
+         */
+        uint64_t               setvol_count;
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {
@@ -114,16 +132,16 @@ typedef struct _client_fd_ctx {
                                             fd's position in the saved_fds list.
                                         */
         int64_t           remote_fd;
-        inode_t          *inode;
-        uint64_t          ino;
-        uint64_t          gen;
         char              is_dir;
         char              released;
         int32_t           flags;
-        int32_t           wbflags;
         fd_lk_ctx_t      *lk_ctx;
         pthread_mutex_t   mutex;
+        lk_heal_state_t   lk_heal_state;
+        uuid_t            gfid;
+        void (*reopen_done) (struct _client_fd_ctx*, xlator_t *);
         struct list_head  lock_list;     /* List of all granted locks on this fd */
+        int32_t           reopen_attempts;
 } clnt_fd_ctx_t;
 
 typedef struct _client_posix_lock {
@@ -144,7 +162,6 @@ typedef struct client_local {
         fd_t                *fd;
         clnt_fd_ctx_t       *fdctx;
         uint32_t             flags;
-        uint32_t             wbflags;
         struct iobref       *iobref;
 
         client_posix_lock_t *client_lock;
@@ -152,18 +169,18 @@ typedef struct client_local {
         int32_t              cmd;
         struct list_head     lock_list;
         pthread_mutex_t      mutex;
+        char                *name;
+        gf_boolean_t         attempt_reopen;
 } clnt_local_t;
 
 typedef struct client_args {
         loc_t              *loc;
         fd_t               *fd;
-        dict_t             *xattr_req;
         const char         *linkname;
         struct iobref      *iobref;
         struct iovec       *vector;
         dict_t             *xattr;
         struct iatt        *stbuf;
-        dict_t             *dict;
         loc_t              *oldloc;
         loc_t              *newloc;
         const char         *name;
@@ -177,7 +194,6 @@ typedef struct client_args {
         mode_t              mode;
         dev_t               rdev;
         int32_t             flags;
-        int32_t             wbflags;
         int32_t             count;
         int32_t             datasync;
         entrylk_cmd         cmd_entrylk;
@@ -185,6 +201,9 @@ typedef struct client_args {
         gf_xattrop_flags_t  optype;
         int32_t             valid;
         int32_t             len;
+
+        mode_t              umask;
+        dict_t             *xdata;
 } clnt_args_t;
 
 typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *args);
@@ -203,9 +222,6 @@ int client_submit_request (xlator_t *this, void *req,
                            struct iovec *rsp_payload, int rsp_count,
                            struct iobref *rsp_iobref, xdrproc_t xdrproc);
 
-int protocol_client_reopendir (xlator_t *this, clnt_fd_ctx_t *fdctx);
-int protocol_client_reopen (xlator_t *this, clnt_fd_ctx_t *fdctx);
-
 int unserialize_rsp_dirent (struct gfs3_readdir_rsp *rsp, gf_dirent_t *entries);
 int unserialize_rsp_direntp (xlator_t *this, fd_t *fd,
                              struct gfs3_readdirp_rsp *rsp, gf_dirent_t *entries);
@@ -216,7 +232,6 @@ int client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx);
 int32_t delete_granted_locks_owner (fd_t *fd, gf_lkowner_t *owner);
 int client_add_lock_for_recovery (fd_t *fd, struct gf_flock *flock,
                                   gf_lkowner_t *owner, int32_t cmd);
-uint64_t decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf);
 int32_t delete_granted_locks_fd (clnt_fd_ctx_t *fdctx);
 int32_t client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd);
 void client_save_number_fds (clnt_conf_t *conf, int count);
@@ -236,4 +251,12 @@ int client_mark_fd_bad (xlator_t *this);
 int client_set_lk_version (xlator_t *this);
 
 int client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t use_try_lock);
+void client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this);
+void client_attempt_reopen (fd_t *fd, xlator_t *this);
+int client_get_remote_fd (xlator_t *this, fd_t *fd, int flags,
+                          int64_t *remote_fd);
+int client_fd_fop_prepare_local (call_frame_t *frame, fd_t *fd,
+                                 int64_t remote_fd);
+gf_boolean_t
+__is_fd_reopen_in_progress (clnt_fd_ctx_t *fdctx);
 #endif /* !_CLIENT_H */
diff --git a/xlators/protocol/legacy/Makefile.am b/xlators/protocol/legacy/Makefile.am
deleted file mode 100644
index 991486302..000000000
--- a/xlators/protocol/legacy/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = lib transport client server
-
-CLEANFILES =
diff --git a/xlators/protocol/legacy/client/src/Makefile.am b/xlators/protocol/legacy/client/src/Makefile.am
deleted file mode 100644
index 2ae64ebd0..000000000
--- a/xlators/protocol/legacy/client/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-
-xlator_LTLIBRARIES = client-old.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/legacy/protocol
-
-client_old_la_LDFLAGS = -module -avoidversion
-
-client_old_la_SOURCES = client-protocol.c saved-frames.c
-
-client_old_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	$(top_builddir)/xlators/protocol/legacy/lib/src/libgfproto.la
-
-noinst_HEADERS = client-protocol.h saved-frames.h client-mem-types.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)  \
-	-I$(top_srcdir)/xlators/protocol/legacy/lib/src
-
-CLEANFILES = 
-
-install-data-hook:
-	ln -sf client-old.so $(DESTDIR)$(xlatordir)/client.so
diff --git a/xlators/protocol/legacy/client/src/client-mem-types.h b/xlators/protocol/legacy/client/src/client-mem-types.h
deleted file mode 100644
index d2e2878c3..000000000
--- a/xlators/protocol/legacy/client/src/client-mem-types.h
+++ /dev/null
@@ -1,43 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __CLIENT_MEM_TYPES_H__
-#define __CLIENT_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_client_mem_types_ {
-        gf_client_mt_dir_entry_t = gf_common_mt_end + 1,
-        gf_client_mt_volfile_ctx,
-        gf_client_mt_client_state_t,
-        gf_client_mt_client_conf_t,
-        gf_client_mt_locker,
-        gf_client_mt_lock_table,
-        gf_client_mt_char,
-        gf_client_mt_client_connection_t,
-        gf_client_mt_client_fd_ctx_t,
-        gf_client_mt_client_local_t,
-        gf_client_mt_saved_frames,
-        gf_client_mt_saved_frame,
-        gf_client_mt_end
-};
-#endif
-
diff --git a/xlators/protocol/legacy/client/src/client-protocol.c b/xlators/protocol/legacy/client/src/client-protocol.c
deleted file mode 100644
index 804b93a5c..000000000
--- a/xlators/protocol/legacy/client/src/client-protocol.c
+++ /dev/null
@@ -1,6683 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-#include <inttypes.h>
-
-
-#include "glusterfs.h"
-#include "client-protocol.h"
-#include "compat.h"
-#include "dict.h"
-#include "protocol.h"
-#include "transport.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "client-mem-types.h"
-
-#include <sys/resource.h>
-#include <inttypes.h>
-
-/* for default_*_cbk functions */
-#include "defaults.c"
-#include "saved-frames.h"
-#include "common-utils.h"
-
-int protocol_client_cleanup (transport_t *trans);
-int protocol_client_interpret (xlator_t *this, transport_t *trans,
-                               char *hdr_p, size_t hdrlen,
-                               struct iobuf *iobuf);
-int
-protocol_client_xfer (call_frame_t *frame, xlator_t *this, transport_t *trans,
-                      int type, int op,
-                      gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iovec *vector, int count,
-                      struct iobref *iobref);
-
-int
-protocol_client_post_handshake (call_frame_t *frame, xlator_t *this);
-
-static gf_op_t gf_fops[GF_PROTO_FOP_MAXVALUE];
-static gf_op_t gf_mops[GF_MOP_MAXVALUE];
-static gf_op_t gf_cbks[GF_CBK_MAXVALUE];
-
-
-transport_t *
-client_channel (xlator_t *this, int id)
-{
-        transport_t              *trans = NULL;
-        client_conf_t            *conf = NULL;
-        int                       i = 0;
-        struct client_connection *conn = NULL;
-
-        conf = this->private;
-
-        trans = conf->transport[id];
-        conn = trans->xl_private;
-
-        if (conn->connected == 1)
-                goto ret;
-
-        for (i = 0; i < CHANNEL_MAX; i++) {
-                trans = conf->transport[i];
-                conn = trans->xl_private;
-                if (conn->connected == 1)
-                        break;
-        }
-
-ret:
-        return trans;
-}
-
-
-client_fd_ctx_t *
-this_fd_del_ctx (fd_t *file, xlator_t *this)
-{
-        int         dict_ret = -1;
-        uint64_t    ctxaddr  = 0;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
-
-        dict_ret = fd_ctx_del (file, this, &ctxaddr);
-
-        if (dict_ret < 0) {
-                ctxaddr = 0;
-        }
-
-out:
-        return (client_fd_ctx_t *)(unsigned long)ctxaddr;
-}
-
-
-client_fd_ctx_t *
-this_fd_get_ctx (fd_t *file, xlator_t *this)
-{
-        int         dict_ret = -1;
-        uint64_t    ctxaddr = 0;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
-
-        dict_ret = fd_ctx_get (file, this, &ctxaddr);
-
-        if (dict_ret < 0) {
-                ctxaddr = 0;
-        }
-
-out:
-        return (client_fd_ctx_t *)(unsigned long)ctxaddr;
-}
-
-
-static void
-this_fd_set_ctx (fd_t *file, xlator_t *this, loc_t *loc, client_fd_ctx_t *ctx)
-{
-        uint64_t oldaddr = 0;
-        int32_t  ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
-
-        ret = fd_ctx_get (file, this, &oldaddr);
-        if (ret >= 0) {
-                if (loc)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s (%"PRId64"): trying duplicate remote fd set. ",
-                                loc->path, loc->inode->ino);
-                else
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%p: trying duplicate remote fd set. ",
-                                file);
-        }
-
-        ret = fd_ctx_set (file, this, (uint64_t)(unsigned long)ctx);
-        if (ret < 0) {
-                if (loc)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s (%"PRId64"): failed to set remote fd",
-                                loc->path, loc->inode->ino);
-                else
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%p: failed to set remote fd",
-                                file);
-        }
-out:
-        return;
-}
-
-
-static int
-client_local_wipe (client_local_t *local)
-{
-        if (local) {
-                loc_wipe (&local->loc);
-
-                if (local->fd)
-                        fd_unref (local->fd);
-
-                mem_put (local);
-        }
-
-        return 0;
-}
-
-/*
- * lookup_frame - lookup call frame corresponding to a given callid
- * @trans: transport object
- * @callid: call id of the frame
- *
- * not for external reference
- */
-
-static call_frame_t *
-lookup_frame (transport_t *trans, int32_t op, int8_t type, int64_t callid)
-{
-        client_connection_t *conn = NULL;
-        call_frame_t        *frame = NULL;
-
-        conn = trans->xl_private;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                frame = saved_frames_get (conn->saved_frames,
-                                          op, type, callid);
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        return frame;
-}
-
-
-static void
-call_bail (void *data)
-{
-        client_connection_t  *conn = NULL;
-        struct timeval        current;
-        transport_t          *trans = NULL;
-        struct list_head      list;
-        struct saved_frame   *saved_frame = NULL;
-        struct saved_frame   *trav = NULL;
-        struct saved_frame   *tmp = NULL;
-        call_frame_t         *frame = NULL;
-        gf_hdr_common_t       hdr = {0, };
-        char                **gf_op_list = NULL;
-        gf_op_t              *gf_ops = NULL;
-        struct tm             frame_sent_tm;
-        char                  frame_sent[32] = {0,};
-        struct timeval        timeout = {0,};
-        gf_timer_cbk_t        timer_cbk = NULL;
-
-        GF_VALIDATE_OR_GOTO ("client", data, out);
-        trans = data;
-
-        conn = trans->xl_private;
-
-        gettimeofday (&current, NULL);
-        INIT_LIST_HEAD (&list);
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                /* Chaining to get call-always functionality from
-                   call-once timer */
-                if (conn->timer) {
-                        timer_cbk = conn->timer->callbk;
-
-                        timeout.tv_sec = 10;
-                        timeout.tv_usec = 0;
-
-                        gf_timer_call_cancel (trans->xl->ctx, conn->timer);
-                        conn->timer = gf_timer_call_after (trans->xl->ctx,
-                                                           timeout,
-                                                           timer_cbk,
-                                                           trans);
-                        if (conn->timer == NULL) {
-                                gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                        "Cannot create bailout timer");
-                        }
-                }
-
-                do {
-                        saved_frame =
-                                saved_frames_get_timedout (conn->saved_frames,
-                                                           GF_OP_TYPE_MOP_REQUEST,
-                                                           conn->frame_timeout,
-                                                           &current);
-                        if (saved_frame)
-                                list_add (&saved_frame->list, &list);
-
-                } while (saved_frame);
-
-                do {
-                        saved_frame =
-                                saved_frames_get_timedout (conn->saved_frames,
-                                                           GF_OP_TYPE_FOP_REQUEST,
-                                                           conn->frame_timeout,
-                                                           &current);
-                        if (saved_frame)
-                                list_add (&saved_frame->list, &list);
-                } while (saved_frame);
-
-                do {
-                        saved_frame =
-                                saved_frames_get_timedout (conn->saved_frames,
-                                                           GF_OP_TYPE_CBK_REQUEST,
-                                                           conn->frame_timeout,
-                                                           &current);
-                        if (saved_frame)
-                                list_add (&saved_frame->list, &list);
-                } while (saved_frame);
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        hdr.rsp.op_ret   = hton32 (-1);
-        hdr.rsp.op_errno = hton32 (ENOTCONN);
-
-        list_for_each_entry_safe (trav, tmp, &list, list) {
-                switch (trav->type)
-                {
-                case GF_OP_TYPE_FOP_REQUEST:
-                        gf_ops = gf_fops;
-                        gf_op_list = gf_fop_list;
-                        break;
-                case GF_OP_TYPE_MOP_REQUEST:
-                        gf_ops = gf_mops;
-                        gf_op_list = gf_mop_list;
-                        break;
-                case GF_OP_TYPE_CBK_REQUEST:
-                        gf_ops = gf_cbks;
-                        gf_op_list = gf_cbk_list;
-                        break;
-                default:
-                        goto out;
-                }
-
-                localtime_r (&trav->saved_at.tv_sec, &frame_sent_tm);
-                strftime (frame_sent, 32, "%Y-%m-%d %H:%M:%S", &frame_sent_tm);
-
-                gf_log (trans->xl->name, GF_LOG_ERROR,
-                        "bailing out frame %s(%d) "
-                        "frame sent = %s. frame-timeout = %d",
-                        gf_op_list[trav->op], trav->op,
-                        frame_sent, conn->frame_timeout);
-
-                hdr.type = hton32 (trav->type);
-                hdr.op   = hton32 (trav->op);
-
-                frame = trav->frame;
-
-                gf_ops[trav->op] (frame, &hdr, sizeof (hdr), NULL);
-
-                list_del_init (&trav->list);
-                GF_FREE (trav);
-        }
-out:
-        return;
-}
-
-
-void
-save_frame (transport_t *trans, call_frame_t *frame,
-            int32_t op, int8_t type, uint64_t callid)
-{
-        client_connection_t *conn = NULL;
-        struct timeval       timeout = {0, };
-
-
-        conn = trans->xl_private;
-
-        saved_frames_put (conn->saved_frames, frame, op, type, callid);
-
-        if (conn->timer == NULL && conn->frame_timeout) {
-                timeout.tv_sec  = 10;
-                timeout.tv_usec = 0;
-                conn->timer = gf_timer_call_after (trans->xl->ctx, timeout,
-                                                   call_bail, (void *) trans);
-        }
-}
-
-
-
-void
-client_ping_timer_expired (void *data)
-{
-        xlator_t            *this = NULL;
-        transport_t         *trans = NULL;
-        client_conf_t       *conf = NULL;
-        client_connection_t *conn = NULL;
-        int                  disconnect = 0;
-        int                  transport_activity = 0;
-        struct timeval       timeout = {0, };
-        struct timeval       current = {0, };
-
-        trans = data;
-        this  = trans->xl;
-        conf  = this->private;
-        conn  = trans->xl_private;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->ping_timer)
-                        gf_timer_call_cancel (trans->xl->ctx,
-                                              conn->ping_timer);
-                gettimeofday (&current, NULL);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        if (((current.tv_sec - conf->last_received.tv_sec) <
-                             conn->ping_timeout)
-                            || ((current.tv_sec - conf->last_sent.tv_sec) <
-                                conn->ping_timeout)) {
-                                transport_activity = 1;
-                        }
-                }
-                pthread_mutex_unlock (&conf->mutex);
-
-                if (transport_activity) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "ping timer expired but transport activity "
-                                "detected - not bailing transport");
-                        conn->transport_activity = 0;
-                        timeout.tv_sec = conn->ping_timeout;
-                        timeout.tv_usec = 0;
-
-                        conn->ping_timer =
-                                gf_timer_call_after (trans->xl->ctx, timeout,
-                                                     client_ping_timer_expired,
-                                                     (void *) trans);
-                        if (conn->ping_timer == NULL)
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "unable to setup timer");
-
-                } else {
-                        conn->ping_started = 0;
-                        conn->ping_timer = NULL;
-                        disconnect = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-        if (disconnect) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Server %s has not responded in the last %d "
-                        "seconds, disconnecting.",
-                        conf->transport[0]->peerinfo.identifier,
-                        conn->ping_timeout);
-
-                transport_disconnect (conf->transport[0]);
-                transport_disconnect (conf->transport[1]);
-        }
-}
-
-
-void
-client_start_ping (void *data)
-{
-        xlator_t            *this = NULL;
-        transport_t         *trans = NULL;
-        client_conf_t       *conf = NULL;
-        client_connection_t *conn = NULL;
-        int32_t              ret = -1;
-        gf_hdr_common_t     *hdr = NULL;
-        struct timeval       timeout = {0, };
-        call_frame_t        *dummy_frame = NULL;
-        size_t               hdrlen = -1;
-        gf_mop_ping_req_t   *req = NULL;
-        int                  frame_count = 0;
-
-
-        trans = data;
-        this  = trans->xl;
-        conf  = this->private;
-        conn  = trans->xl_private;
-
-        if (!conn->ping_timeout)
-                return;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->ping_timer)
-                        gf_timer_call_cancel (trans->xl->ctx, conn->ping_timer);
-
-                conn->ping_timer = NULL;
-                conn->ping_started = 0;
-
-                if (conn->saved_frames)
-                        /* treat the case where conn->saved_frames is NULL
-                           as no pending frames */
-                        frame_count = conn->saved_frames->count;
-
-                if ((frame_count == 0) || !conn->connected) {
-                        /* using goto looked ugly here,
-                         * hence getting out this way */
-                        /* unlock */
-                        pthread_mutex_unlock (&conn->lock);
-                        return;
-                }
-
-                if (frame_count < 0) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "saved_frames->count is %"PRId64,
-                                conn->saved_frames->count);
-                        conn->saved_frames->count = 0;
-                }
-
-                timeout.tv_sec = conn->ping_timeout;
-                timeout.tv_usec = 0;
-
-                conn->ping_timer =
-                        gf_timer_call_after (trans->xl->ctx, timeout,
-                                             client_ping_timer_expired,
-                                             (void *) trans);
-
-                if (conn->ping_timer == NULL) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "unable to setup timer");
-                } else {
-                        conn->ping_started = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        if (!hdr)
-                goto err;
-
-        dummy_frame = create_frame (this, this->ctx->pool);
-
-        if (!dummy_frame)
-                goto err;
-
-        dummy_frame->local = trans;
-
-        ret = protocol_client_xfer (dummy_frame, this, trans,
-                                    GF_OP_TYPE_MOP_REQUEST, GF_MOP_PING,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return;
-err:
-        if (hdr)
-                GF_FREE (hdr);
-
-        if (dummy_frame)
-                STACK_DESTROY (dummy_frame->root);
-
-        return;
-}
-
-
-int
-client_ping_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        xlator_t            *this = NULL;
-        transport_t         *trans = NULL;
-        client_connection_t *conn = NULL;
-        struct timeval       timeout = {0, };
-        int                  op_ret = 0;
-
-        trans  = frame->local; frame->local = NULL;
-        this   = trans->xl;
-        conn   = trans->xl_private;
-
-        op_ret = ntoh32 (hdr->rsp.op_ret);
-
-        if (op_ret == -1) {
-                /* timer expired and transport bailed out */
-                gf_log (this->name, GF_LOG_DEBUG, "timer must have expired");
-                goto out;
-        }
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                timeout.tv_sec  = conn->ping_timeout;
-                timeout.tv_usec = 0;
-
-                gf_timer_call_cancel (trans->xl->ctx,
-                                      conn->ping_timer);
-
-                conn->ping_timer =
-                        gf_timer_call_after (trans->xl->ctx, timeout,
-                                             client_start_ping, (void *)trans);
-                if (conn->ping_timer == NULL)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "gf_timer_call_after() returned NULL");
-        }
-        pthread_mutex_unlock (&conn->lock);
-out:
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-int
-client_encode_groups (call_frame_t *frame, gf_hdr_common_t *hdr)
-{
-        int     i = 0;
-        if ((!frame) || (!hdr))
-                return -1;
-
-        hdr->req.ngrps = hton32 (frame->root->ngrps);
-        if (frame->root->ngrps == 0)
-                return 0;
-
-        for (; i < frame->root->ngrps; ++i)
-                hdr->req.groups[i] = hton32 (frame->root->groups[i]);
-
-        return 0;
-}
-
-
-int
-protocol_client_xfer (call_frame_t *frame, xlator_t *this, transport_t *trans,
-                      int type, int op,
-                      gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iovec *vector, int count,
-                      struct iobref *iobref)
-{
-        client_conf_t        *conf = NULL;
-        client_connection_t  *conn = NULL;
-        uint64_t              callid = 0;
-        int32_t               ret = -1;
-        int                   start_ping = 0;
-        gf_hdr_common_t       rsphdr = {0, };
-
-        conf  = this->private;
-
-        if (!trans) {
-                /* default to bulk op since it is 'safer' */
-                trans = conf->transport[CHANNEL_BULK];
-        }
-        conn  = trans->xl_private;
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                callid = ++conn->callid;
-
-                hdr->callid = hton64 (callid);
-                hdr->op     = hton32 (op);
-                hdr->type   = hton32 (type);
-
-                if (frame) {
-                        hdr->req.uid = hton32 (frame->root->uid);
-                        hdr->req.gid = hton32 (frame->root->gid);
-                        hdr->req.pid = hton32 (frame->root->pid);
-                        hdr->req.lk_owner = hton64 (frame->root->lk_owner);
-                        client_encode_groups (frame, hdr);
-                }
-
-                if (conn->connected == 0)
-                        transport_connect (trans);
-
-                ret = -1;
-
-                if (conn->connected ||
-                    ((type == GF_OP_TYPE_MOP_REQUEST) &&
-                     (op == GF_MOP_SETVOLUME))) {
-                        ret = transport_submit (trans, (char *)hdr, hdrlen,
-                                                vector, count, iobref);
-                }
-
-                if ((ret >= 0) && frame) {
-                        pthread_mutex_lock (&conf->mutex);
-                        {
-                                gettimeofday (&conf->last_sent, NULL);
-                        }
-                        pthread_mutex_unlock (&conf->mutex);
-                        save_frame (trans, frame, op, type, callid);
-                }
-
-                if (!conn->ping_started && (ret >= 0)) {
-                        start_ping = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (start_ping)
-                client_start_ping ((void *) trans);
-
-        if (frame && (ret < 0)) {
-                rsphdr.op = op;
-                rsphdr.rsp.op_ret   = hton32 (-1);
-                rsphdr.rsp.op_errno = hton32 (ENOTCONN);
-
-                if (type == GF_OP_TYPE_FOP_REQUEST) {
-                        rsphdr.type = GF_OP_TYPE_FOP_REPLY;
-                        gf_fops[op] (frame, &rsphdr, sizeof (rsphdr), NULL);
-                } else if (type == GF_OP_TYPE_MOP_REQUEST) {
-                        rsphdr.type = GF_OP_TYPE_MOP_REPLY;
-                        gf_mops[op] (frame, &rsphdr, sizeof (rsphdr), NULL);
-                } else {
-                        rsphdr.type = GF_OP_TYPE_CBK_REPLY;
-                        gf_cbks[op] (frame, &rsphdr, sizeof (rsphdr), NULL);
-                }
-
-                GF_FREE (hdr);
-        }
-
-        return ret;
-}
-
-
-
-/**
- * client_create - create function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @path: complete path to file
- * @flags: create flags
- * @mode: create mode
- *
- * external reference through client_protocol_xlator->fops->create
- */
-
-int
-client_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-               mode_t mode, fd_t *fd, dict_t *params)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_create_req_t *req = NULL;
-        size_t               hdrlen = 0;
-        size_t               pathlen = 0;
-        size_t               baselen = 0;
-        int32_t              ret = -1;
-        ino_t                par = 0;
-        uint64_t             gen = 0;
-        client_local_t      *local = NULL;
-
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        local->fd = fd_ref (fd);
-        loc_copy (&local->loc, loc);
-        local->flags = flags;
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "CREATE %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent inode",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->flags   = hton32 (gf_flags_from_flags (flags));
-        req->mode    = hton32 (mode);
-        req->par     = hton64 (par);
-        req->gen     = hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_CREATE,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, fd, NULL, NULL);
-        return 0;
-
-}
-
-/**
- * client_open - open function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location of file
- * @flags: open flags
- * @mode: open modes
- *
- * external reference through client_protocol_xlator->fops->open
- */
-
-int
-client_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-             fd_t *fd, int32_t wbflags)
-{
-        int                 ret = -1;
-        gf_hdr_common_t    *hdr = NULL;
-        size_t              hdrlen = 0;
-        gf_fop_open_req_t  *req = NULL;
-        size_t              pathlen = 0;
-        ino_t               ino = 0;
-        uint64_t            gen = 0;
-        client_local_t     *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        local->fd = fd_ref (fd);
-        loc_copy (&local->loc, loc);
-        local->flags = flags;
-        local->wbflags = wbflags;
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "OPEN %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        req->flags = hton32 (gf_flags_from_flags (flags));
-        req->wbflags = hton32 (wbflags);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_OPEN,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, fd);
-        return 0;
-
-}
-
-
-/**
- * client_stat - stat function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- *
- * external reference through client_protocol_xlator->fops->stat
- */
-
-int
-client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_stat_req_t *req = NULL;
-        size_t             hdrlen = -1;
-        int32_t            ret = -1;
-        size_t             pathlen = 0;
-        ino_t              ino = 0;
-        ino_t              gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "STAT %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_STAT,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_readlink - readlink function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- * @size:
- *
- * external reference through client_protocol_xlator->fops->readlink
- */
-int
-client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_readlink_req_t *req = NULL;
-        size_t                 hdrlen = -1;
-        int                    ret = -1;
-        size_t                 pathlen = 0;
-        ino_t                  ino = 0;
-        uint64_t               gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "READLINK %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        req->size = hton32 (size);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_READLINK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND_STRICT (readlink, frame, -1, EINVAL,
-                             NULL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_mknod - mknod function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @path: pathname of node
- * @mode:
- * @dev:
- *
- * external reference through client_protocol_xlator->fops->mknod
- */
-int
-client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t dev, dict_t *params)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_mknod_req_t *req = NULL;
-        size_t              hdrlen = -1;
-        int                 ret = -1;
-        size_t              pathlen = 0;
-        size_t              baselen = 0;
-        ino_t               par = 0;
-        uint64_t            gen = 0;
-        client_local_t     *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "MKNOD %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par  = hton64 (par);
-        req->gen  = hton64 (gen);
-        req->mode = hton32 (mode);
-        req->dev  = hton64 (dev);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_MKNOD,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, loc->inode, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_mkdir - mkdir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @path: pathname of directory
- * @mode:
- *
- * external reference through client_protocol_xlator->fops->mkdir
- */
-int
-client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dict_t *params)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_mkdir_req_t *req = NULL;
-        size_t              hdrlen = -1;
-        int                 ret = -1;
-        size_t              pathlen = 0;
-        size_t              baselen = 0;
-        ino_t               par = 0;
-        uint64_t            gen = 0;
-        client_local_t     *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "MKDIR %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par  = hton64 (par);
-        req->gen  = hton64 (gen);
-        req->mode = hton32 (mode);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_MKDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, loc->inode, NULL);
-        return 0;
-
-}
-
-/**
- * client_unlink - unlink function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location of file
- *
- * external reference through client_protocol_xlator->fops->unlink
- */
-
-int
-client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_unlink_req_t *req = NULL;
-        size_t               hdrlen = -1;
-        int                  ret = -1;
-        size_t               pathlen = 0;
-        size_t               baselen = 0;
-        ino_t                par = 0;
-        uint64_t             gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "UNLINK %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par  = hton64 (par);
-        req->gen  = hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_UNLINK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-/**
- * client_rmdir - rmdir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- *
- * external reference through client_protocol_xlator->fops->rmdir
- */
-
-int
-client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_rmdir_req_t *req = NULL;
-        size_t              hdrlen = -1;
-        int                 ret = -1;
-        size_t              pathlen = 0;
-        size_t              baselen = 0;
-        ino_t               par = 0;
-        uint64_t            gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "RMDIR %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen);
-        hdr    = gf_hdr_new (req, pathlen + baselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par  = hton64 (par);
-        req->gen  = hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_RMDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-
-/**
- * client_symlink - symlink function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @oldpath: pathname of target
- * @newpath: pathname of symlink
- *
- * external reference through client_protocol_xlator->fops->symlink
- */
-
-int
-client_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-                loc_t *loc, dict_t *params)
-{
-        int                   ret = -1;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_symlink_req_t *req = NULL;
-        size_t                hdrlen  = 0;
-        size_t                pathlen = 0;
-        size_t                newlen  = 0;
-        size_t                baselen = 0;
-        ino_t                 par = 0;
-        uint64_t              gen = 0;
-        client_local_t       *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-
-        frame->local = local;
-
-        pathlen = STRLEN_0 (loc->path);
-        baselen = STRLEN_0 (loc->name);
-        newlen = STRLEN_0 (linkname);
-        ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-        if (loc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "SYMLINK %"PRId64"/%s (%s): failed to get remote inode"
-                        " number parent",
-                        loc->parent->ino, loc->name, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen + newlen);
-        hdr    = gf_hdr_new (req, pathlen + baselen + newlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->par =  hton64 (par);
-        req->gen =  hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->bname + pathlen, loc->name);
-        strcpy (req->linkname + pathlen + baselen, linkname);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_SYMLINK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, loc->inode, NULL);
-        return 0;
-
-}
-
-/**
- * client_rename - rename function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @oldloc: location of old pathname
- * @newloc: location of new pathname
- *
- * external reference through client_protocol_xlator->fops->rename
- */
-
-int
-client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
-{
-        int                  ret = -1;
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_rename_req_t *req = NULL;
-        size_t               hdrlen = 0;
-        size_t               oldpathlen = 0;
-        size_t               oldbaselen = 0;
-        size_t               newpathlen = 0;
-        size_t               newbaselen = 0;
-        ino_t                oldpar = 0;
-        uint64_t             oldgen = 0;
-        ino_t                newpar = 0;
-        uint64_t             newgen = 0;
-
-        oldpathlen = STRLEN_0 (oldloc->path);
-        oldbaselen = STRLEN_0 (oldloc->name);
-        newpathlen = STRLEN_0 (newloc->path);
-        newbaselen = STRLEN_0 (newloc->name);
-        ret = inode_ctx_get2 (oldloc->parent, this, &oldpar, &oldgen);
-        if (oldloc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "RENAME %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for source parent",
-                        oldloc->parent->ino, oldloc->name, oldloc->path);
-                goto unwind;
-        }
-
-        ret = inode_ctx_get2 (newloc->parent, this, &newpar, &newgen);
-        if (newloc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "CREATE %"PRId64"/%s (%s): failed to get remote inode "
-                        "number for destination parent",
-                        newloc->parent->ino, newloc->name, newloc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, (oldpathlen + oldbaselen +
-                                   newpathlen + newbaselen));
-        hdr    = gf_hdr_new (req, (oldpathlen + oldbaselen +
-                                   newpathlen + newbaselen));
-
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->oldpar = hton64 (oldpar);
-        req->oldgen = hton64 (oldgen);
-        req->newpar = hton64 (newpar);
-        req->newgen = hton64 (newgen);
-
-        strcpy (req->oldpath, oldloc->path);
-        strcpy (req->oldbname + oldpathlen, oldloc->name);
-        strcpy (req->newpath  + oldpathlen + oldbaselen, newloc->path);
-        strcpy (req->newbname + oldpathlen + oldbaselen + newpathlen,
-                newloc->name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_RENAME,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-/**
- * client_link - link function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @oldloc: location of old pathname
- * @newpath: new pathname
- *
- * external reference through client_protocol_xlator->fops->link
- */
-
-int
-client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
-{
-        int                ret = -1;
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_link_req_t *req = NULL;
-        size_t             hdrlen = 0;
-        size_t             oldpathlen = 0;
-        size_t             newpathlen = 0;
-        size_t             newbaselen = 0;
-        ino_t              oldino = 0;
-        uint64_t           oldgen = 0;
-        ino_t              newpar = 0;
-        uint64_t           newgen = 0;
-        client_local_t    *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, oldloc);
-
-        frame->local = local;
-
-        oldpathlen = STRLEN_0 (oldloc->path);
-        newpathlen = STRLEN_0 (newloc->path);
-        newbaselen = STRLEN_0 (newloc->name);
-
-        ret = inode_ctx_get2 (oldloc->inode, this, &oldino, &oldgen);
-        if (oldloc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "LINK %"PRId64"/%s (%s) ==> %"PRId64" (%s): "
-                        "failed to get remote inode number for source inode",
-                        newloc->parent->ino, newloc->name, newloc->path,
-                        oldloc->ino, oldloc->path);
-                goto unwind;
-        }
-
-        ret = inode_ctx_get2 (newloc->parent, this, &newpar, &newgen);
-        if (newloc->parent->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "LINK %"PRId64"/%s (%s) ==> %"PRId64" (%s): "
-                        "failed to get remote inode number destination parent",
-                        newloc->parent->ino, newloc->name, newloc->path,
-                        oldloc->ino, oldloc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, oldpathlen + newpathlen + newbaselen);
-        hdr    = gf_hdr_new (req, oldpathlen + newpathlen + newbaselen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        strcpy (req->oldpath, oldloc->path);
-        strcpy (req->newpath  + oldpathlen, newloc->path);
-        strcpy (req->newbname + oldpathlen + newpathlen, newloc->name);
-
-        req->oldino = hton64 (oldino);
-        req->oldgen = hton64 (oldgen);
-        req->newpar = hton64 (newpar);
-        req->newgen = hton64 (newgen);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_LINK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, oldloc->inode, NULL);
-        return 0;
-}
-
-
-/**
- * client_truncate - truncate function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- * @offset:
- *
- * external reference through client_protocol_xlator->fops->truncate
- */
-
-int
-client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_truncate_req_t *req = NULL;
-        size_t                 hdrlen = -1;
-        int                    ret = -1;
-        size_t                 pathlen = 0;
-        ino_t                  ino = 0;
-        uint64_t               gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "TRUNCATE %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino    = hton64 (ino);
-        req->gen    = hton64 (gen);
-        req->offset = hton64 (offset);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_TRUNCATE,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_readv - readv function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @size:
- * @offset:
- *
- * external reference through client_protocol_xlator->fops->readv
- */
-
-int
-client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_read_req_t  *req = NULL;
-        size_t              hdrlen = 0;
-        int64_t             remote_fd = -1;
-        int                 ret = -1;
-        client_fd_ctx_t    *fdctx = NULL;
-        client_conf_t      *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx, EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL, 0, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx, EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL, 0, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd     = hton64 (remote_fd);
-        req->size   = hton32 (size);
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_READ,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL, 0, NULL);
-        return 0;
-
-}
-
-/**
- * client_writev - writev function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @vector:
- * @count:
- * @offset:
- *
- * external reference through client_protocol_xlator->fops->writev
- */
-
-int
-client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               struct iovec *vector, int32_t count, off_t offset,
-               struct iobref *iobref)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_write_req_t *req = NULL;
-        size_t              hdrlen = 0;
-        int64_t             remote_fd = -1;
-        int                 ret = -1;
-        client_fd_ctx_t    *fdctx = NULL;
-        client_conf_t      *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd     = hton64 (remote_fd);
-        req->size   = hton32 (iov_length (vector, count));
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_WRITE,
-                                    hdr, hdrlen, vector, count, iobref);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_statfs - statfs function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- *
- * external reference through client_protocol_xlator->fops->statfs
- */
-
-int
-client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_statfs_req_t *req = NULL;
-        size_t               hdrlen = -1;
-        int                  ret = -1;
-        size_t               pathlen = 0;
-        ino_t                ino = 0;
-        ino_t                gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        if (loc->inode) {
-                ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-                if (loc->inode->ino && ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "STATFS %"PRId64" (%s): "
-                                "failed to get remote inode number",
-                                loc->inode->ino, loc->path);
-                        goto unwind;
-                }
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino = hton64 (ino);
-        req->gen = hton64 (gen);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_STATFS,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_flush - flush function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- *
- * external reference through client_protocol_xlator->fops->flush
- */
-
-int
-client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_flush_req_t  *req = NULL;
-        size_t               hdrlen = 0;
-        int64_t              remote_fd = -1;
-        int                  ret = -1;
-        client_fd_ctx_t     *fdctx = NULL;
-        client_conf_t       *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd = hton64 (remote_fd);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FLUSH,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-/**
- * client_fsync - fsync function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @flags:
- *
- * external reference through client_protocol_xlator->fops->fsync
- */
-
-int
-client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_fsync_req_t *req = NULL;
-        size_t              hdrlen = 0;
-        int64_t             remote_fd = -1;
-        int32_t             ret = -1;
-        client_fd_ctx_t    *fdctx = NULL;
-        client_conf_t      *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd   = hton64 (remote_fd);
-        req->data = hton32 (flags);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSYNC,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-int
-client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                gf_xattrop_flags_t flags, dict_t *dict)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_xattrop_req_t *req = NULL;
-        size_t                hdrlen = 0;
-        size_t                dict_len = 0;
-        int32_t               ret = -1;
-        size_t                pathlen = 0;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-        char                 *buf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("client", this, unwind);
-
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-
-        if (dict) {
-                ret = dict_allocate_and_serialize (dict, &buf, &dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict(%p)",
-                                dict);
-                        goto unwind;
-                }
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "XATTROP %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, dict_len + pathlen);
-        hdr    = gf_hdr_new (req, dict_len + pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->flags = hton32 (flags);
-        req->dict_len = hton32 (dict_len);
-        if (dict) {
-                memcpy (req->dict, buf, dict_len);
-                GF_FREE (buf);
-        }
-
-        req->ino = hton64 (ino);
-        req->gen = hton64 (gen);
-        strcpy (req->path + dict_len, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_XATTROP,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-int
-client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 gf_xattrop_flags_t flags, dict_t *dict)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fxattrop_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 dict_len = 0;
-        int64_t                remote_fd = -1;
-        int32_t                ret = -1;
-        ino_t                  ino = 0;
-        client_fd_ctx_t       *fdctx = NULL;
-        client_conf_t         *conf  = NULL;
-
-        conf = this->private;
-
-        if (dict) {
-                dict_len = dict_serialized_length (dict);
-                if (dict_len < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict(%p)",
-                                dict);
-                        goto unwind;
-                }
-        }
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. "
-                        "returning EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        ino = fd->inode->ino;
-        remote_fd = fdctx->remote_fd;
-
-        hdrlen = gf_hdr_len (req, dict_len);
-        hdr    = gf_hdr_new (req, dict_len);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->flags = hton32 (flags);
-        req->dict_len = hton32 (dict_len);
-        if (dict) {
-                ret = dict_serialize (dict, req->dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to serialize dictionary(%p)",
-                                dict);
-                        goto unwind;
-                }
-        }
-        req->fd = hton64 (remote_fd);
-        req->ino = hton64 (ino);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FXATTROP,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EBADFD, NULL);
-        return 0;
-
-}
-
-/**
- * client_setxattr - setxattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location
- * @dict: dictionary which contains key:value to be set.
- * @flags:
- *
- * external reference through client_protocol_xlator->fops->setxattr
- */
-
-int
-client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 dict_t *dict, int32_t flags)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_setxattr_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 dict_len = 0;
-        int                    ret = -1;
-        size_t                 pathlen = 0;
-        ino_t                  ino = 0;
-        uint64_t               gen = 0;
-
-        dict_len = dict_serialized_length (dict);
-        if (dict_len < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict(%p)",
-                        dict);
-                goto unwind;
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "SETXATTR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, dict_len + pathlen);
-        hdr    = gf_hdr_new (req, dict_len + pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        req->flags = hton32 (flags);
-        req->dict_len = hton32 (dict_len);
-
-        ret = dict_serialize (dict, req->dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to serialize dictionary(%p)",
-                        dict);
-                goto unwind;
-        }
-
-        strcpy (req->path + dict_len, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_SETXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-/**
- * client_fsetxattr - fsetxattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: fd
- * @dict: dictionary which contains key:value to be set.
- * @flags:
- *
- * external reference through client_protocol_xlator->fops->fsetxattr
- */
-
-int
-client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int32_t flags)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_fsetxattr_req_t *req = NULL;
-        size_t                  hdrlen = 0;
-        size_t                  dict_len = 0;
-        ino_t                   ino;
-        int                     ret = -1;
-        int64_t                 remote_fd = -1;
-        client_fd_ctx_t        *fdctx = NULL;
-        client_conf_t          *conf  = NULL;
-
-        conf = this->private;
-
-        dict_len = dict_serialized_length (dict);
-        if (dict_len < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict(%p)",
-                        dict);
-                goto unwind;
-        }
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        ino = fd->inode->ino;
-        remote_fd = fdctx->remote_fd;
-
-        hdrlen = gf_hdr_len (req, dict_len);
-        hdr    = gf_hdr_new (req, dict_len);
-
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->fd    = hton64 (remote_fd);
-        req->flags = hton32 (flags);
-        req->dict_len = hton32 (dict_len);
-
-        ret = dict_serialize (dict, req->dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to serialize dictionary(%p)",
-                        dict);
-                goto unwind;
-        }
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSETXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-/**
- * client_getxattr - getxattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location structure
- *
- * external reference through client_protocol_xlator->fops->getxattr
- */
-
-int
-client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name)
-{
-        int                    ret = -1;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_getxattr_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 pathlen = 0;
-        size_t                 namelen = 0;
-        ino_t                  ino = 0;
-        uint64_t               gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        if (name)
-                namelen = STRLEN_0 (name);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "GETXATTR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + namelen);
-        hdr    = gf_hdr_new (req, pathlen + namelen);
-        GF_VALIDATE_OR_GOTO (frame->this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        req->namelen = hton32 (namelen);
-        strcpy (req->path, loc->path);
-        if (name)
-                strcpy (req->name + pathlen, name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_GETXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-/**
- * client_fgetxattr - fgetxattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: fd
- *
- * external reference through client_protocol_xlator->fops->fgetxattr
- */
-
-int
-client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  const char *name)
-{
-        int                     ret = -1;
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_fgetxattr_req_t *req = NULL;
-        size_t                  hdrlen = 0;
-        int64_t                 remote_fd = -1;
-        size_t                  namelen = 0;
-        ino_t                   ino = 0;
-        client_fd_ctx_t        *fdctx = NULL;
-        client_conf_t          *conf  = NULL;
-
-        if (name)
-                namelen = STRLEN_0 (name);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get remote fd. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        ino = fd->inode->ino;
-        remote_fd = fdctx->remote_fd;
-
-        hdrlen = gf_hdr_len (req, namelen);
-        hdr    = gf_hdr_new (req, namelen);
-
-        GF_VALIDATE_OR_GOTO (frame->this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->fd    = hton64 (remote_fd);
-        req->namelen = hton32 (namelen);
-
-        if (name)
-                strcpy (req->name, name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FGETXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-/**
- * client_removexattr - removexattr function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location structure
- * @name:
- *
- * external reference through client_protocol_xlator->fops->removexattr
- */
-
-int
-client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name)
-{
-        int                       ret = -1;
-        gf_hdr_common_t          *hdr = NULL;
-        gf_fop_removexattr_req_t *req = NULL;
-        size_t                    hdrlen = 0;
-        size_t                    namelen = 0;
-        size_t                    pathlen = 0;
-        ino_t                     ino = 0;
-        uint64_t                  gen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        namelen = STRLEN_0 (name);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "REMOVEXATTR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + namelen);
-        hdr    = gf_hdr_new (req, pathlen + namelen);
-        GF_VALIDATE_OR_GOTO (frame->this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        strcpy (req->path, loc->path);
-        strcpy (req->name + pathlen, name);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_REMOVEXATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-/**
- * client_opendir - opendir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location structure
- *
- * external reference through client_protocol_xlator->fops->opendir
- */
-
-int
-client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                fd_t *fd)
-{
-        gf_fop_opendir_req_t *req = NULL;
-        gf_hdr_common_t      *hdr = NULL;
-        size_t                hdrlen = 0;
-        int                   ret = -1;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-        size_t                pathlen = 0;
-        client_local_t       *local = NULL;
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-        local->fd = fd_ref (fd);
-
-        frame->local = local;
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "OPENDIR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (frame->this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino = hton64 (ino);
-        req->gen = hton64 (gen);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_OPENDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, fd);
-        return 0;
-
-}
-
-/**
- * client_readdirp - readdirp function for client protocol
- * @frame: call frame
- * @this: this translator structure
- *
- * external reference through client_protocol_xlator->fops->readdirp
- */
-
-int
-client_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset)
-{
-        gf_hdr_common_t         *hdr = NULL;
-        gf_fop_readdirp_req_t   *req = NULL;
-        size_t                  hdrlen = 0;
-        int64_t                 remote_fd = -1;
-        int                     ret = -1;
-        client_fd_ctx_t         *fdctx = NULL;
-        client_conf_t           *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req->fd     = hton64 (remote_fd);
-        req->size   = hton32 (size);
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_READDIRP,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EBADFD, NULL);
-        return 0;
-
-}
-
-
-/**
- * client_readdir - readdir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- *
- * external reference through client_protocol_xlator->fops->readdir
- */
-
-int
-client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_readdir_req_t  *req = NULL;
-        size_t                 hdrlen = 0;
-        int64_t                remote_fd = -1;
-        int                    ret = -1;
-        client_fd_ctx_t       *fdctx = NULL;
-        client_conf_t         *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req->fd     = hton64 (remote_fd);
-        req->size   = hton32 (size);
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_READDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EBADFD, NULL);
-        return 0;
-
-}
-
-/**
- * client_fsyncdir - fsyncdir function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @flags:
- *
- * external reference through client_protocol_xlator->fops->fsyncdir
- */
-
-int
-client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fsyncdir_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        int64_t                remote_fd = -1;
-        int32_t                ret = -1;
-        client_fd_ctx_t       *fdctx = NULL;
-        client_conf_t         *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                goto unwind;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->data = hton32 (flags);
-        req->fd   = hton64 (remote_fd);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSYNCDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        STACK_UNWIND (frame, -1, EBADFD);
-        return 0;
-}
-
-/**
- * client_access - access function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @loc: location structure
- * @mode:
- *
- * external reference through client_protocol_xlator->fops->access
- */
-
-int
-client_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_access_req_t *req = NULL;
-        size_t               hdrlen = -1;
-        int                  ret = -1;
-        ino_t                ino = 0;
-        uint64_t             gen = 0;
-        size_t               pathlen = 0;
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "ACCESS %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        req->mask = hton32 (mask);
-        strcpy (req->path, loc->path);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_ACCESS,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-/**
- * client_ftrucate - ftruncate function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @offset: offset to truncate to
- *
- * external reference through client_protocol_xlator->fops->ftruncate
- */
-
-int
-client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  off_t offset)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_ftruncate_req_t *req = NULL;
-        int64_t                 remote_fd = -1;
-        size_t                  hdrlen = -1;
-        int                     ret = -1;
-        client_fd_ctx_t        *fdctx = NULL;
-        client_conf_t          *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd     = hton64 (remote_fd);
-        req->offset = hton64 (offset);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FTRUNCATE,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-/**
- * client_fstat - fstat function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- *
- * external reference through client_protocol_xlator->fops->fstat
- */
-
-int
-client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_fstat_req_t *req = NULL;
-        int64_t             remote_fd = -1;
-        size_t              hdrlen = -1;
-        int                 ret = -1;
-        client_fd_ctx_t    *fdctx = NULL;
-        client_conf_t      *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                goto unwind;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd = hton64 (remote_fd);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSTAT,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-
-}
-
-/**
- * client_lk - lk function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @fd: file descriptor structure
- * @cmd: lock command
- * @lock:
- *
- * external reference through client_protocol_xlator->fops->lk
- */
-
-int
-client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *flock)
-{
-        int              ret = -1;
-        gf_hdr_common_t *hdr = NULL;
-        gf_fop_lk_req_t *req = NULL;
-        size_t           hdrlen = 0;
-        int64_t          remote_fd = -1;
-        int32_t          gf_cmd = 0;
-        int32_t          gf_type = 0;
-        client_fd_ctx_t *fdctx = NULL;
-        client_conf_t   *conf  = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "(%"PRId64"): failed to get"
-                        " fd ctx. EBADFD", fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        if (cmd == F_GETLK || cmd == F_GETLK64)
-                gf_cmd = GF_LK_GETLK;
-        else if (cmd == F_SETLK || cmd == F_SETLK64)
-                gf_cmd = GF_LK_SETLK;
-        else if (cmd == F_SETLKW || cmd == F_SETLKW64)
-                gf_cmd = GF_LK_SETLKW;
-        else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unknown cmd (%d)!", gf_cmd);
-                goto unwind;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd   = hton64 (remote_fd);
-        req->cmd  = hton32 (gf_cmd);
-        req->type = hton32 (gf_type);
-        gf_flock_from_flock (&req->flock, flock);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_LK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-/**
- * client_inodelk - inodelk function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @inode: inode structure
- * @cmd: lock command
- * @lock: flock struct
- *
- * external reference through client_protocol_xlator->fops->inodelk
- */
-
-int
-client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                loc_t *loc, int32_t cmd, struct gf_flock *flock)
-{
-        int                   ret = -1;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_inodelk_req_t *req = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_cmd = 0;
-        int32_t               gf_type = 0;
-        ino_t                 ino  = 0;
-        uint64_t              gen  = 0;
-        size_t                pathlen = 0;
-        size_t                vollen  = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        vollen  = STRLEN_0 (volume);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "INODELK %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        if (cmd == F_GETLK || cmd == F_GETLK64)
-                gf_cmd = GF_LK_GETLK;
-        else if (cmd == F_SETLK || cmd == F_SETLK64)
-                gf_cmd = GF_LK_SETLK;
-        else if (cmd == F_SETLKW || cmd == F_SETLKW64)
-                gf_cmd = GF_LK_SETLKW;
-        else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unknown cmd (%d)!", gf_cmd);
-                goto unwind;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + vollen);
-        hdr    = gf_hdr_new (req, pathlen + vollen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        strcpy (req->path, loc->path);
-        strcpy (req->path + pathlen, volume);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-
-        req->cmd  = hton32 (gf_cmd);
-        req->type = hton32 (gf_type);
-        gf_flock_from_flock (&req->flock, flock);
-
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST,
-                                    GF_PROTO_FOP_INODELK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-
-/**
- * client_finodelk - finodelk function for client protocol
- * @frame: call frame
- * @this: this translator structure
- * @inode: inode structure
- * @cmd: lock command
- * @lock: flock struct
- *
- * external reference through client_protocol_xlator->fops->finodelk
- */
-
-int
-client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                 fd_t *fd, int32_t cmd, struct gf_flock *flock)
-{
-        int                    ret = -1;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_finodelk_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 vollen = 0;
-        int32_t                gf_cmd = 0;
-        int32_t                gf_type = 0;
-        int64_t                remote_fd = -1;
-        client_fd_ctx_t       *fdctx = NULL;
-        client_conf_t         *conf  = NULL;
-
-        vollen = STRLEN_0 (volume);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        if (cmd == F_GETLK || cmd == F_GETLK64)
-                gf_cmd = GF_LK_GETLK;
-        else if (cmd == F_SETLK || cmd == F_SETLK64)
-                gf_cmd = GF_LK_SETLK;
-        else if (cmd == F_SETLKW || cmd == F_SETLKW64)
-                gf_cmd = GF_LK_SETLKW;
-        else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unknown cmd (%d)!", gf_cmd);
-                goto unwind;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        hdrlen = gf_hdr_len (req, vollen);
-        hdr    = gf_hdr_new (req, vollen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        strcpy (req->volume, volume);
-
-        req->fd = hton64 (remote_fd);
-
-        req->cmd  = hton32 (gf_cmd);
-        req->type = hton32 (gf_type);
-        gf_flock_from_flock (&req->flock, flock);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST,
-                                    GF_PROTO_FOP_FINODELK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-
-int
-client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                loc_t *loc, const char *name, entrylk_cmd cmd,
-                entrylk_type type)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_entrylk_req_t *req = NULL;
-        size_t                pathlen = 0;
-        size_t                vollen  = 0;
-        size_t                hdrlen = -1;
-        int                   ret = -1;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-        size_t                namelen = 0;
-
-        pathlen = STRLEN_0 (loc->path);
-        vollen  = STRLEN_0 (volume);
-
-        if (name)
-                namelen = STRLEN_0 (name);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "ENTRYLK %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + vollen + namelen);
-        hdr    = gf_hdr_new (req, pathlen + vollen + namelen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        req->namelen = hton64 (namelen);
-
-        strcpy (req->path, loc->path);
-        if (name)
-                strcpy (req->name + pathlen, name);
-        strcpy (req->volume + pathlen + namelen, volume);
-
-        req->cmd  = hton32 (cmd);
-        req->type = hton32 (type);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_ENTRYLK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-
-}
-
-
-int
-client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                 fd_t *fd, const char *name, entrylk_cmd cmd,
-                 entrylk_type type)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_fentrylk_req_t  *req = NULL;
-        int64_t                 remote_fd = -1;
-        size_t                  vollen  = 0;
-        size_t                  namelen = 0;
-        size_t                  hdrlen = -1;
-        int                     ret = -1;
-        client_fd_ctx_t        *fdctx = NULL;
-        client_conf_t          *conf  = NULL;
-
-        if (name)
-                namelen = STRLEN_0 (name);
-
-        conf = this->private;
-
-        vollen = STRLEN_0 (volume);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, namelen + vollen);
-        hdr    = gf_hdr_new (req, namelen + vollen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd = hton64 (remote_fd);
-        req->namelen = hton64 (namelen);
-
-        if (name)
-                strcpy (req->name, name);
-
-        strcpy (req->volume + namelen, volume);
-
-        req->cmd  = hton32 (cmd);
-        req->type = hton32 (type);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FENTRYLK,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-
-        STACK_UNWIND (frame, -1, EINVAL);
-        return 0;
-}
-
-/*
- * client_lookup - lookup function for client protocol
- * @frame: call frame
- * @this:
- * @loc: location
- *
- * not for external reference
- */
-
-int
-client_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xattr_req)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_lookup_req_t *req = NULL;
-        size_t               hdrlen = -1;
-        int                  ret = -1;
-        ino_t                ino = 0;
-        ino_t                par = 0;
-        uint64_t             gen = 0;
-        size_t               dictlen = 0;
-        size_t               pathlen = 0;
-        size_t               baselen = 0;
-        int32_t              op_ret = -1;
-        int32_t              op_errno = EINVAL;
-        client_local_t      *local = NULL;
-        char                *buf = NULL;
-
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->path, unwind);
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
-
-        loc_copy (&local->loc, loc);
-
-        frame->local = local;
-
-        if (loc->ino != 1 && loc->parent) {
-                ret = inode_ctx_get2 (loc->parent, this, &par, &gen);
-                if (loc->parent->ino && ret < 0) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "LOOKUP %"PRId64"/%s (%s): failed to get "
-                                "remote inode number for parent",
-                                loc->parent->ino, loc->name, loc->path);
-                        goto unwind;
-                }
-                GF_VALIDATE_OR_GOTO (this->name, loc->name, unwind);
-                baselen = STRLEN_0 (loc->name);
-        } else {
-                ino = 1;
-        }
-
-        pathlen = STRLEN_0 (loc->path);
-
-        if (xattr_req) {
-                ret = dict_allocate_and_serialize (xattr_req, &buf, &dictlen);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to get serialized length of dict(%p)",
-                                xattr_req);
-                        goto unwind;
-                }
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen + baselen + dictlen);
-        hdr    = gf_hdr_new (req, pathlen + baselen + dictlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (ino);
-        req->gen   = hton64 (gen);
-        req->par   = hton64 (par);
-        strcpy (req->path, loc->path);
-        if (baselen)
-                strcpy (req->path + pathlen, loc->name);
-
-        if (dictlen > 0) {
-                memcpy (req->dict + pathlen + baselen, buf, dictlen);
-                GF_FREE (buf);
-        }
-
-        req->dictlen = hton32 (dictlen);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_LOOKUP,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-
-unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, (loc)?loc->inode:NULL, NULL, NULL);
-        return ret;
-}
-
-
-int
-client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_setattr_req_t *req = NULL;
-        size_t                hdrlen = 0;
-        size_t                pathlen = 0;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-        int                   ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("client", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-
-        pathlen = STRLEN_0 (loc->path);
-
-        ret = inode_ctx_get2 (loc->inode, this, &ino, &gen);
-        if (loc->inode->ino && ret < 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "SETATTR %"PRId64" (%s): "
-                        "failed to get remote inode number",
-                        loc->inode->ino, loc->path);
-                goto unwind;
-        }
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->ino  = hton64 (ino);
-        req->gen  = hton64 (gen);
-        strcpy (req->path, loc->path);
-
-        gf_stat_from_iatt (&req->stbuf, stbuf);
-        req->valid = hton32 (valid);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_SETATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-int
-client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fsetattr_req_t *req = NULL;
-        size_t                 hdrlen = 0;
-        int                    ret = -1;
-        client_fd_ctx_t       *fdctx = NULL;
-        int64_t                remote_fd = -1;
-        client_conf_t         *conf  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("client", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req    = gf_param (hdr);
-
-        req->fd = hton64 (remote_fd);
-
-        gf_stat_from_iatt (&req->stbuf, stbuf);
-        req->valid = hton32 (valid);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_FSETATTR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
-        return 0;
-}
-
-
-int
-client_fdctx_destroy (xlator_t *this, client_fd_ctx_t *fdctx)
-{
-        call_frame_t             *fr = NULL;
-        int32_t                   ret = -1;
-        gf_hdr_common_t          *hdr = NULL;
-        size_t                    hdrlen = 0;
-        gf_cbk_release_req_t     *req = NULL;
-        gf_cbk_releasedir_req_t  *reqdir = NULL;
-        int64_t                   remote_fd = -1;
-        int                       op = 0;
-
-        remote_fd = fdctx->remote_fd;
-
-        if (remote_fd == -1)
-                goto out;
-
-        if (fdctx->is_dir) {
-                hdrlen     = gf_hdr_len (reqdir, 0);
-                hdr        = gf_hdr_new (reqdir, 0);
-                op         = GF_CBK_RELEASEDIR;
-                reqdir     = gf_param (hdr);
-                reqdir->fd = hton64 (remote_fd);
-        } else {
-                hdrlen  = gf_hdr_len (req, 0);
-                hdr     = gf_hdr_new (req, 0);
-                op      = GF_CBK_RELEASE;
-                req     = gf_param (hdr);
-                req->fd = hton64 (remote_fd);
-        }
-
-        fr = create_frame (this, this->ctx->pool);
-
-        ret = protocol_client_xfer (fr, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_CBK_REQUEST, op,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-out:
-        inode_unref (fdctx->inode);
-        GF_FREE (fdctx);
-
-        return ret;
-}
-
-
-/**
- * client_releasedir - releasedir function for client protocol
- * @this: this translator structure
- * @fd: file descriptor structure
- *
- * external reference through client_protocol_xlator->cbks->releasedir
- */
-
-int
-client_releasedir (xlator_t *this, fd_t *fd)
-{
-        int64_t                remote_fd = -1;
-        client_conf_t         *conf = NULL;
-        client_fd_ctx_t       *fdctx = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_del_ctx (fd, this);
-                if (fdctx != NULL) {
-                        remote_fd = fdctx->remote_fd;
-
-                        /* fdctx->remote_fd == -1 indicates a reopen attempt
-                           in progress. Just mark ->released = 1 and let
-                           reopen_cbk handle releasing
-                        */
-
-                        if (remote_fd != -1)
-                                list_del_init (&fdctx->sfd_pos);
-
-                        fdctx->released = 1;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (remote_fd != -1)
-                client_fdctx_destroy (this, fdctx);
-
-        return 0;
-}
-
-
-/**
- * client_release - release function for client protocol
- * @this: this translator structure
- * @fd: file descriptor structure
- *
- * external reference through client_protocol_xlator->cbks->release
- *
- */
-int
-client_release (xlator_t *this, fd_t *fd)
-{
-        int64_t                remote_fd = -1;
-        client_conf_t         *conf = NULL;
-        client_fd_ctx_t       *fdctx = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_del_ctx (fd, this);
-                if (fdctx != NULL) {
-                        remote_fd = fdctx->remote_fd;
-
-                        /* fdctx->remote_fd == -1 indicates a reopen attempt
-                           in progress. Just mark ->released = 1 and let
-                           reopen_cbk handle releasing
-                        */
-
-                        if (remote_fd != -1)
-                                list_del_init (&fdctx->sfd_pos);
-
-                        fdctx->released = 1;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (remote_fd != -1)
-                client_fdctx_destroy (this, fdctx);
-
-        return 0;
-}
-
-/*
- * MGMT_OPS
- */
-
-/* Callbacks */
-
-int
-client_fxattrop_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_xattrop_rsp_t *rsp = NULL;
-        int32_t               op_ret   = 0;
-        int32_t               gf_errno = 0;
-        int32_t               op_errno = 0;
-        int32_t               dict_len = 0;
-        dict_t               *dict = NULL;
-        int32_t               ret = -1;
-        char                 *dictbuf = NULL;
-
-        rsp = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (frame->this->name, rsp, fail);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-
-        if (op_ret >= 0) {
-                op_ret = -1;
-                dict_len = ntoh32 (rsp->dict_len);
-
-                if (dict_len > 0) {
-                        dictbuf = memdup (rsp->dict, dict_len);
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dictbuf, fail);
-
-                        dict = dict_new();
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dict, fail);
-
-                        ret = dict_unserialize (dictbuf, dict_len, &dict);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "failed to serialize dictionary(%p)",
-                                        dict);
-                                op_errno = -ret;
-                                goto fail;
-                        } else {
-                                dict->extra_free = dictbuf;
-                                dictbuf = NULL;
-                        }
-                }
-                op_ret = 0;
-        }
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-
-fail:
-        STACK_UNWIND (frame, op_ret, op_errno, dict);
-
-        if (dictbuf)
-                GF_FREE (dictbuf);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
-}
-
-
-int
-client_xattrop_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_xattrop_rsp_t *rsp = NULL;
-        int32_t               op_ret   = -1;
-        int32_t               gf_errno = EINVAL;
-        int32_t               op_errno = 0;
-        int32_t               dict_len = 0;
-        dict_t               *dict = NULL;
-        int32_t               ret = -1;
-        char                 *dictbuf = NULL;
-
-        rsp = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (frame->this->name, rsp, fail);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        if (op_ret >= 0) {
-                op_ret = -1;
-                dict_len = ntoh32 (rsp->dict_len);
-
-                if (dict_len > 0) {
-                        dictbuf = memdup (rsp->dict, dict_len);
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dictbuf, fail);
-
-                        dict = get_new_dict();
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dict, fail);
-                        dict_ref (dict);
-
-                        ret = dict_unserialize (dictbuf, dict_len, &dict);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "failed to serialize dictionary(%p)",
-                                        dict);
-                                goto fail;
-                        } else {
-                                dict->extra_free = dictbuf;
-                                dictbuf = NULL;
-                        }
-                }
-                op_ret = 0;
-        }
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-
-
-fail:
-        STACK_UNWIND (frame, op_ret, op_errno, dict);
-
-        if (dictbuf)
-                GF_FREE (dictbuf);
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
-}
-
-/*
- * client_create_cbk - create callback function for client protocol
- * @frame: call frame
- * @args: arguments in dictionary
- *
- * not for external reference
- */
-
-int
-client_create_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_fop_create_rsp_t  *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-        fd_t                 *fd = NULL;
-        inode_t              *inode = NULL;
-        struct iatt           stbuf = {0, };
-        struct iatt           preparent = {0, };
-        struct iatt           postparent = {0, };
-        int64_t               remote_fd = 0;
-        int32_t               ret = -1;
-        client_local_t       *local = NULL;
-        client_conf_t        *conf = NULL;
-        client_fd_ctx_t      *fdctx = NULL;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-
-        local = frame->local; frame->local = NULL;
-        conf  = frame->this->private;
-        fd    = local->fd;
-        inode = local->loc.inode;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0) {
-                remote_fd = ntoh64 (rsp->fd);
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-
-                ino = stbuf.ia_ino;
-                gen = stbuf.ia_gen;
-        }
-
-        if (op_ret >= 0) {
-                ret = inode_ctx_put2 (local->loc.inode, frame->this, ino, gen);
-
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "CREATE %"PRId64"/%s (%s): failed to set "
-                                "remote inode number to inode ctx",
-                                local->loc.parent->ino, local->loc.name,
-                                local->loc.path);
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        goto unwind_out;
-                }
-
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_client_fd_ctx_t);
-                if (!fdctx) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind_out;
-                }
-
-                fdctx->remote_fd = remote_fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->ino       = ino;
-                fdctx->gen       = gen;
-                fdctx->flags     = local->flags;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->mutex);
-        }
-unwind_out:
-        STACK_UNWIND (frame, op_ret, op_errno, fd, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-
-/*
- * client_open_cbk - open callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-int
-client_open_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        int32_t              op_ret = -1;
-        int32_t              op_errno = ENOTCONN;
-        fd_t                *fd = NULL;
-        int64_t              remote_fd = 0;
-        gf_fop_open_rsp_t   *rsp = NULL;
-        client_local_t      *local = NULL;
-        client_conf_t       *conf = NULL;
-        client_fd_ctx_t     *fdctx = NULL;
-        ino_t                ino = 0;
-        uint64_t             gen = 0;
-
-
-        local = frame->local;
-
-        if (local->op) {
-                local->op (frame, hdr, hdrlen, iobuf);
-                return 0;
-        }
-
-        frame->local = NULL;
-        conf  = frame->this->private;
-        fd    = local->fd;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0) {
-                remote_fd = ntoh64 (rsp->fd);
-        }
-
-        if (op_ret >= 0) {
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_client_fd_ctx_t);
-                if (!fdctx) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind_out;
-                }
-
-                inode_ctx_get2 (fd->inode, frame->this, &ino, &gen);
-
-                fdctx->remote_fd = remote_fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->ino       = ino;
-                fdctx->gen       = gen;
-                fdctx->flags     = local->flags;
-                fdctx->wbflags   = local->wbflags;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->mutex);
-        }
-unwind_out:
-        STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_stat_cbk - stat callback for client protocol
- * @frame: call frame
- * @args: arguments dictionary
- *
- * not for external reference
- */
-
-int
-client_stat_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        struct iatt        stbuf = {0, };
-        gf_fop_stat_rsp_t *rsp = NULL;
-        int32_t            op_ret = 0;
-        int32_t            op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &stbuf);
-
-        return 0;
-}
-
-
-/*
- * client_mknod_cbk - mknod callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_mknod_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_mknod_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iatt         stbuf = {0, };
-        inode_t            *inode = NULL;
-        client_local_t     *local = NULL;
-        int                 ret = 0;
-        struct iatt         preparent = {0,};
-        struct iatt         postparent = {0,};
-
-        local = frame->local;
-        frame->local = NULL;
-        inode = local->loc.inode;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                ret = inode_ctx_put2 (local->loc.inode, frame->this,
-                                      stbuf.ia_ino, stbuf.ia_gen);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "MKNOD %"PRId64"/%s (%s): failed to set remote"
-                                " inode number to inode ctx",
-                                local->loc.parent->ino, local->loc.name,
-                                local->loc.path);
-
-                        STACK_UNWIND (frame, -1, EINVAL, inode, NULL,
-                                      NULL, NULL);
-                        return 0;
-                }
-
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_symlink_cbk - symlink callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_symlink_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_symlink_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-        struct iatt           stbuf = {0, };
-        struct iatt           preparent = {0,};
-        struct iatt           postparent = {0,};
-        inode_t              *inode = NULL;
-        client_local_t       *local = NULL;
-        int                   ret = 0;
-
-        local = frame->local;
-        frame->local = NULL;
-        inode = local->loc.inode;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                ret = inode_ctx_put2 (inode, frame->this,
-                                      stbuf.ia_ino, stbuf.ia_gen);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "SYMLINK %"PRId64"/%s (%s): failed to set "
-                                "remote inode number to inode ctx",
-                                local->loc.parent->ino, local->loc.name,
-                                local->loc.path);
-                        STACK_UNWIND (frame, -1, EINVAL, inode, NULL,
-                                      NULL, NULL);
-                        return 0;
-                }
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_link_cbk - link callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_link_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_link_rsp_t *rsp = NULL;
-        int32_t            op_ret = 0;
-        int32_t            op_errno = 0;
-        struct iatt        stbuf = {0, };
-        inode_t           *inode = NULL;
-        client_local_t    *local = NULL;
-        struct iatt        preparent = {0,};
-        struct iatt        postparent = {0,};
-
-        local = frame->local;
-        frame->local = NULL;
-        inode = local->loc.inode;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_truncate_cbk - truncate callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_truncate_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_truncate_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-        struct iatt         prestat = {0, };
-        struct iatt         poststat = {0, };
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->prestat, &prestat);
-                gf_stat_to_iatt (&rsp->poststat, &poststat);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &prestat, &poststat);
-
-        return 0;
-}
-
-/* client_fstat_cbk - fstat callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_fstat_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        struct iatt         stbuf = {0, };
-        gf_fop_fstat_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &stbuf);
-
-        return 0;
-}
-
-/*
- * client_ftruncate_cbk - ftruncate callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-int
-client_ftruncate_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iobuf *iobuf)
-{
-        gf_fop_ftruncate_rsp_t *rsp = NULL;
-        int32_t                 op_ret = 0;
-        int32_t                 op_errno = 0;
-        struct iatt             prestat = {0, };
-        struct iatt             poststat = {0, };
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->prestat, &prestat);
-                gf_stat_to_iatt (&rsp->poststat, &poststat);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &prestat, &poststat);
-
-        return 0;
-}
-
-
-/* client_readv_cbk - readv callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external referece
- */
-
-int
-client_readv_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_read_rsp_t  *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iovec        vector = {0, };
-        struct iatt         stbuf = {0, };
-        struct iobref      *iobref = NULL;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret != -1) {
-                iobref = iobref_new ();
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-                vector.iov_len  = op_ret;
-
-                if (op_ret > 0) {
-                        vector.iov_base = iobuf->ptr;
-                        iobref_add (iobref, iobuf);
-                }
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &vector, 1, &stbuf, iobref);
-
-        if (iobref)
-                iobref_unref (iobref);
-
-        if (iobuf)
-                iobuf_unref (iobuf);
-
-        return 0;
-}
-
-/*
- * client_write_cbk - write callback for client protocol
- * @frame: cal frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_write_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_write_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iatt         prestat = {0, };
-        struct iatt         poststat = {0, };
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->prestat, &prestat);
-                gf_stat_to_iatt (&rsp->poststat, &poststat);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &prestat, &poststat);
-
-        return 0;
-}
-
-
-int
-client_readdirp_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_readdirp_rsp_t   *rsp = NULL;
-        int32_t                 op_ret = 0;
-        int32_t                 op_errno = 0;
-        uint32_t                buf_size = 0;
-        gf_dirent_t             entries;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        INIT_LIST_HEAD (&entries.list);
-        if (op_ret > 0) {
-                buf_size = ntoh32 (rsp->size);
-                gf_dirent_unserialize (&entries, rsp->buf, buf_size);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &entries);
-
-        gf_dirent_free (&entries);
-
-        return 0;
-}
-
-
-int
-client_readdir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_readdir_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-        uint32_t              buf_size = 0;
-        gf_dirent_t           entries;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        INIT_LIST_HEAD (&entries.list);
-        if (op_ret > 0) {
-                buf_size = ntoh32 (rsp->size);
-                gf_dirent_unserialize (&entries, rsp->buf, buf_size);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &entries);
-
-        gf_dirent_free (&entries);
-
-        return 0;
-}
-
-/*
- * client_fsync_cbk - fsync callback for client protocol
- *
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_fsync_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        struct iatt         prestat = {0, };
-        struct iatt         poststat = {0,};
-        gf_fop_fsync_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->prestat, &prestat);
-                gf_stat_to_iatt (&rsp->poststat, &poststat);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &prestat, &poststat);
-
-        return 0;
-}
-
-/*
- * client_unlink_cbk - unlink callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_unlink_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_fop_unlink_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-        struct iatt          preparent = {0,};
-        struct iatt          postparent = {0,};
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &preparent, &postparent);
-
-        return 0;
-}
-
-/*
- * client_rename_cbk - rename callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_rename_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        struct iatt          stbuf = {0, };
-        gf_fop_rename_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-        struct iatt          preoldparent = {0, };
-        struct iatt          postoldparent = {0, };
-        struct iatt          prenewparent = {0, };
-        struct iatt          postnewparent = {0, };
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-                gf_stat_to_iatt (&rsp->preoldparent, &preoldparent);
-                gf_stat_to_iatt (&rsp->postoldparent, &postoldparent);
-                gf_stat_to_iatt (&rsp->prenewparent, &prenewparent);
-                gf_stat_to_iatt (&rsp->postnewparent, &postnewparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &stbuf, &preoldparent,
-                      &postoldparent, &prenewparent, &postnewparent);
-
-        return 0;
-}
-
-
-/*
- * client_readlink_cbk - readlink callback for client protocol
- *
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-int
-client_readlink_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_readlink_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-        char                  *link = NULL;
-        struct iatt            stbuf = {0,};
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret > 0) {
-                link = rsp->path;
-                gf_stat_to_iatt (&rsp->buf, &stbuf);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, link, &stbuf);
-        return 0;
-}
-
-/*
- * client_mkdir_cbk - mkdir callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_mkdir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_mkdir_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iatt         stbuf = {0, };
-        inode_t            *inode = NULL;
-        client_local_t     *local = NULL;
-        int                 ret = 0;
-        struct iatt         preparent = {0,};
-        struct iatt         postparent = {0,};
-
-        local = frame->local;
-        inode = local->loc.inode;
-        frame->local = NULL;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                ret = inode_ctx_put2 (inode, frame->this, stbuf.ia_ino,
-                                      stbuf.ia_gen);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "MKDIR %"PRId64"/%s (%s): failed to set "
-                                "remote inode number to inode ctx",
-                                local->loc.parent->ino, local->loc.name,
-                                local->loc.path);
-                        STACK_UNWIND (frame, -1, EINVAL, inode, NULL,
-                                      NULL, NULL);
-                        return 0;
-                }
-
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf,
-                      &preparent, &postparent);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_flush_cbk - flush callback for client protocol
- *
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_flush_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_opendir_cbk - opendir callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_opendir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        int32_t               op_ret   = -1;
-        int32_t               op_errno = ENOTCONN;
-        fd_t                 *fd       = NULL;
-        int64_t               remote_fd = 0;
-        gf_fop_opendir_rsp_t *rsp       = NULL;
-        client_local_t       *local = NULL;
-        client_conf_t        *conf = NULL;
-        client_fd_ctx_t      *fdctx = NULL;
-        ino_t                 ino = 0;
-        uint64_t              gen = 0;
-
-
-        local = frame->local;
-
-        if (local->op) {
-                local->op (frame, hdr, hdrlen, iobuf);
-                return 0;
-        }
-
-        frame->local = NULL;
-        conf  = frame->this->private;
-        fd    = local->fd;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0) {
-                remote_fd = ntoh64 (rsp->fd);
-        }
-
-        if (op_ret >= 0) {
-                fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                                   gf_client_mt_client_fd_ctx_t);
-                if (!fdctx) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind_out;
-                }
-
-                inode_ctx_get2 (fd->inode, frame->this, &ino, &gen);
-
-                fdctx->remote_fd = remote_fd;
-                fdctx->inode     = inode_ref (fd->inode);
-                fdctx->ino       = ino;
-                fdctx->gen       = gen;
-
-                fdctx->is_dir    = 1;
-
-                INIT_LIST_HEAD (&fdctx->sfd_pos);
-
-                this_fd_set_ctx (fd, frame->this, &local->loc, fdctx);
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                }
-                pthread_mutex_unlock (&conf->mutex);
-        }
-unwind_out:
-        STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-/*
- * client_rmdir_cbk - rmdir callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_rmdir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_rmdir_rsp_t *rsp = NULL;
-        int32_t             op_ret = 0;
-        int32_t             op_errno = 0;
-        struct iatt         preparent = {0,};
-        struct iatt         postparent = {0,};
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->preparent, &preparent);
-                gf_stat_to_iatt (&rsp->postparent, &postparent);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &preparent, &postparent);
-
-        return 0;
-}
-
-/*
- * client_access_cbk - access callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_access_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_fop_access_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_lookup_cbk - lookup callback for client protocol
- *
- * @frame: call frame
- * @args: arguments dictionary
- *
- * not for external reference
- */
-
-int
-client_lookup_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        struct iatt          stbuf = {0, };
-        struct iatt          postparent = {0, };
-        inode_t             *inode = NULL;
-        dict_t              *xattr = NULL;
-        gf_fop_lookup_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-        size_t               dict_len = 0;
-        char                *dictbuf = NULL;
-        int32_t              ret = -1;
-        int32_t              gf_errno = 0;
-        client_local_t      *local = NULL;
-        ino_t                oldino = 0;
-        uint64_t             oldgen = 0;
-
-        local = frame->local;
-        inode = local->loc.inode;
-        frame->local = NULL;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-
-        gf_stat_to_iatt (&rsp->postparent, &postparent);
-
-        if (op_ret == 0) {
-                op_ret = -1;
-                gf_stat_to_iatt (&rsp->stat, &stbuf);
-
-                ret = inode_ctx_get2 (inode, frame->this, &oldino, &oldgen);
-                if (oldino != stbuf.ia_ino || oldgen != stbuf.ia_gen) {
-                        if (oldino) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "LOOKUP %"PRId64"/%s (%s): "
-                                        "inode number changed from "
-                                        "{%"PRId64",%"PRId64"} to {%"PRId64",%"PRId64"}",
-                                        local->loc.parent ?
-                                        local->loc.parent->ino : (uint64_t) 0,
-                                        local->loc.name,
-                                        local->loc.path,
-                                        oldgen, oldino, stbuf.ia_gen, stbuf.ia_ino);
-                                op_errno = ESTALE;
-                                goto fail;
-                        }
-
-                        ret = inode_ctx_put2 (inode, frame->this,
-                                              stbuf.ia_ino, stbuf.ia_gen);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "LOOKUP %"PRId64"/%s (%s) : "
-                                        "failed to set remote inode "
-                                        "number to inode ctx",
-                                        local->loc.parent ?
-                                        local->loc.parent->ino : (uint64_t) 0,
-                                        local->loc.name,
-                                        local->loc.path);
-                                op_errno = EINVAL;
-                                goto fail;
-                        }
-                }
-
-                dict_len = ntoh32 (rsp->dict_len);
-
-                if (dict_len > 0) {
-                        dictbuf = memdup (rsp->dict, dict_len);
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dictbuf, fail);
-
-                        xattr = dict_new();
-                        GF_VALIDATE_OR_GOTO (frame->this->name, xattr, fail);
-
-                        ret = dict_unserialize (dictbuf, dict_len, &xattr);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "%s (%"PRId64"): failed to "
-                                        "unserialize dictionary",
-                                        local->loc.path, inode->ino);
-                                goto fail;
-                        } else {
-                                xattr->extra_free = dictbuf;
-                                dictbuf = NULL;
-                        }
-                }
-                op_ret = 0;
-        }
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-
-fail:
-        STACK_UNWIND (frame, op_ret, op_errno, inode, &stbuf, xattr,
-                      &postparent);
-
-        client_local_wipe (local);
-
-        if (dictbuf)
-                GF_FREE (dictbuf);
-
-        if (xattr)
-                dict_unref (xattr);
-
-        return 0;
-}
-
-static int32_t
-client_setattr_cbk (call_frame_t *frame,gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        struct iatt           statpre = {0, };
-        struct iatt           statpost = {0, };
-        gf_fop_setattr_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->statpre, &statpre);
-                gf_stat_to_iatt (&rsp->statpost, &statpost);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &statpre, &statpost);
-
-        return 0;
-}
-
-static int32_t
-client_fsetattr_cbk (call_frame_t *frame,gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        struct iatt           statpre = {0, };
-        struct iatt           statpost = {0, };
-        gf_fop_setattr_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_to_iatt (&rsp->statpre, &statpre);
-                gf_stat_to_iatt (&rsp->statpost, &statpost);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &statpre, &statpost);
-
-        return 0;
-}
-
-
-int
-gf_free_direntry (dir_entry_t *head)
-{
-        dir_entry_t *prev = NULL;
-        dir_entry_t *trav = NULL;
-
-        prev = head;
-        GF_VALIDATE_OR_GOTO ("client-protocol", prev, fail);
-
-        trav = head->next;
-        while (trav) {
-                prev->next = trav->next;
-                GF_FREE (trav->name);
-                if (IA_ISLNK (trav->buf.ia_type))
-                        GF_FREE (trav->link);
-                GF_FREE (trav);
-                trav = prev->next;
-        }
-        GF_FREE (head);
-fail:
-        return 0;
-}
-
-/*
- * client_statfs_cbk - statfs callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_statfs_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        struct statvfs       stbuf = {0, };
-        gf_fop_statfs_rsp_t *rsp = NULL;
-        int32_t              op_ret = 0;
-        int32_t              op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret == 0) {
-                gf_statfs_to_statfs (&rsp->statfs, &stbuf);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &stbuf);
-
-        return 0;
-}
-
-/*
- * client_fsyncdir_cbk - fsyncdir callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_fsyncdir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_setxattr_cbk - setxattr callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_setxattr_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_setxattr_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_getxattr_cbk - getxattr callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_getxattr_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_getxattr_rsp_t *rsp = NULL;
-        int32_t                op_ret   = 0;
-        int32_t                gf_errno = 0;
-        int32_t                op_errno = 0;
-        int32_t                dict_len = 0;
-        dict_t                *dict = NULL;
-        int32_t                ret = -1;
-        char                  *dictbuf = NULL;
-        client_local_t        *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-
-        rsp = gf_param (hdr);
-        GF_VALIDATE_OR_GOTO (frame->this->name, rsp, fail);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-
-        if (op_ret >= 0) {
-                op_ret = -1;
-                dict_len = ntoh32 (rsp->dict_len);
-
-                if (dict_len > 0) {
-                        dictbuf = memdup (rsp->dict, dict_len);
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dictbuf, fail);
-
-                        dict = dict_new();
-                        GF_VALIDATE_OR_GOTO (frame->this->name, dict, fail);
-
-                        ret = dict_unserialize (dictbuf, dict_len, &dict);
-                        if (ret < 0) {
-                                gf_log (frame->this->name, GF_LOG_DEBUG,
-                                        "%s (%"PRId64"): failed to "
-                                        "unserialize xattr dictionary",
-                                        local->loc.path,
-                                        local->loc.inode->ino);
-                                goto fail;
-                        } else {
-                                dict->extra_free = dictbuf;
-                                dictbuf = NULL;
-                        }
-                }
-                op_ret = 0;
-        }
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-fail:
-        STACK_UNWIND (frame, op_ret, op_errno, dict);
-
-        client_local_wipe (local);
-
-        if (dictbuf)
-                GF_FREE (dictbuf);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
-}
-
-/*
- * client_removexattr_cbk - removexattr callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_removexattr_cbk (call_frame_t *frame, gf_hdr_common_t *hdr,
-                        size_t hdrlen, struct iobuf *iobuf)
-{
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-/*
- * client_lk_cbk - lk callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_lk_common_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iobuf *iobuf)
-{
-        struct gf_flock     lock = {0,};
-        gf_fop_lk_rsp_t *rsp = NULL;
-        int32_t          op_ret = 0;
-        int32_t          op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret >= 0) {
-                gf_flock_to_flock (&rsp->flock, &lock);
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, &lock);
-        return 0;
-}
-
-/*
- * client_gf_file_lk_cbk - gf_file_lk callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_inodelk_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_inodelk_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-        return 0;
-}
-
-
-int
-client_finodelk_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_finodelk_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-        return 0;
-}
-
-/*
- * client_entrylk_cbk - entrylk callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_entrylk_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_entrylk_rsp_t *rsp = NULL;
-        int32_t               op_ret = 0;
-        int32_t               op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-        return 0;
-}
-
-int
-client_fentrylk_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_fop_fentrylk_rsp_t *rsp = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-        return 0;
-}
-
-
-
-/*
- * client_getspec - getspec function for client protocol
- * @frame: call frame
- * @this: client protocol xlator structure
- * @flag:
- *
- * external reference through client_protocol_xlator->fops->getspec
- */
-
-int
-client_getspec (call_frame_t *frame, xlator_t *this, const char *key,
-                int32_t flag)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_mop_getspec_req_t *req = NULL;
-        size_t                hdrlen = -1;
-        int                   keylen = 0;
-        int                   ret = -1;
-
-        if (key)
-                keylen = STRLEN_0 (key);
-
-        hdrlen = gf_hdr_len (req, keylen);
-        hdr    = gf_hdr_new (req, keylen);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, unwind);
-
-        req        = gf_param (hdr);
-        req->flags = hton32 (flag);
-        req->keylen = hton32 (keylen);
-        if (keylen)
-                strcpy (req->key, key);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_MOP_REQUEST, GF_MOP_GETSPEC,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-unwind:
-        if (hdr)
-                GF_FREE (hdr);
-        STACK_UNWIND (frame, -1, EINVAL, NULL);
-        return 0;
-}
-
-/*
- * client_getspec_cbk - getspec callback for client protocol
- *
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_getspec_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_mop_getspec_rsp_t  *rsp = NULL;
-        char                  *spec_data = NULL;
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-        int32_t                gf_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-        rsp = gf_param (hdr);
-
-        if (op_ret >= 0) {
-                spec_data = rsp->spec;
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, spec_data);
-        return 0;
-}
-
-
-int
-client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                  int32_t len)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_rchecksum_req_t *req = NULL;
-        size_t                  hdrlen = -1;
-        int                     ret = -1;
-
-        int64_t                remote_fd = -1;
-        client_fd_ctx_t       *fdctx     = NULL;
-        client_conf_t         *conf = NULL;
-
-        hdrlen = gf_hdr_len (req, 0);
-        hdr    = gf_hdr_new (req, 0);
-        req    = gf_param (hdr);
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx == NULL) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, 0, NULL);
-                return 0;
-        }
-
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "(%"PRId64"): failed to get fd ctx. EBADFD",
-                        fd->inode->ino);
-                STACK_UNWIND (frame, -1, EBADFD, 0, NULL);
-                return 0;
-        }
-
-        remote_fd = fdctx->remote_fd;
-
-        req->fd     = hton64 (remote_fd);
-        req->offset = hton64 (offset);
-        req->len    = hton32 (len);
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_BULK),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_RCHECKSUM,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-}
-
-
-int
-client_rchecksum_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iobuf *iobuf)
-{
-        gf_fop_rchecksum_rsp_t *rsp = NULL;
-
-        int32_t                op_ret = 0;
-        int32_t                op_errno = 0;
-        int32_t                gf_errno = 0;
-        uint32_t               weak_checksum = 0;
-        unsigned char         *strong_checksum = NULL;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        gf_errno = ntoh32 (hdr->rsp.op_errno);
-        op_errno = gf_error_to_errno (gf_errno);
-
-        if (op_ret >= 0) {
-                weak_checksum   = rsp->weak_checksum;
-                strong_checksum = rsp->strong_checksum;
-        }
-
-        STACK_UNWIND (frame, op_ret, op_errno, weak_checksum, strong_checksum);
-
-        return 0;
-}
-
-
-/*
- * client_setspec_cbk - setspec callback for client protocol
- * @frame: call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_setspec_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        STACK_UNWIND (frame, op_ret, op_errno);
-
-        return 0;
-}
-
-
-
-int
-protocol_client_reopendir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr,
-                               size_t hdrlen, struct iobuf *iobuf)
-{
-        int32_t              op_ret = -1;
-        int32_t              op_errno = ENOTCONN;
-        int64_t              remote_fd = -1;
-        gf_fop_open_rsp_t   *rsp = NULL;
-        client_local_t      *local = NULL;
-        client_conf_t       *conf = NULL;
-        client_fd_ctx_t     *fdctx = NULL;
-
-
-        local = frame->local; frame->local = NULL;
-        conf  = frame->this->private;
-        fdctx = local->fdctx;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0)
-                remote_fd = ntoh64 (rsp->fd);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "reopendir on %s returned %d (%"PRId64")",
-                local->loc.path, op_ret, remote_fd);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx->remote_fd = remote_fd;
-
-                if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        fdctx = NULL;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
-
-        STACK_DESTROY (frame->root);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-
-
-int
-protocol_client_reopendir (xlator_t *this, client_fd_ctx_t *fdctx)
-{
-        int                    ret = -1;
-        gf_hdr_common_t       *hdr = NULL;
-        size_t                 hdrlen = 0;
-        gf_fop_opendir_req_t  *req = NULL;
-        size_t                 pathlen = 0;
-        client_local_t        *local = NULL;
-        inode_t               *inode = NULL;
-        char                  *path = NULL;
-        call_frame_t          *frame = NULL;
-
-        inode = fdctx->inode;
-
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        if (!local) {
-                goto out;
-        }
-
-        local->fdctx = fdctx;
-        local->op = protocol_client_reopendir_cbk;
-        local->loc.path = path; path = NULL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                goto out;
-        }
-
-        pathlen = STRLEN_0 (local->loc.path);
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (fdctx->ino);
-        req->gen   = hton64 (fdctx->gen);
-
-        strcpy (req->path, local->loc.path);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "attempting reopendir on %s", local->loc.path);
-
-        frame->local = local; local = NULL;
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_OPENDIR,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-
-out:
-        if (frame)
-                STACK_DESTROY (frame->root);
-
-        if (local)
-                client_local_wipe (local);
-
-        if (path)
-                GF_FREE (path);
-
-        return 0;
-}
-
-
-int
-protocol_client_reopen_cbk (call_frame_t *frame, gf_hdr_common_t *hdr,
-                            size_t hdrlen, struct iobuf *iobuf)
-{
-        int32_t              op_ret = -1;
-        int32_t              op_errno = ENOTCONN;
-        int64_t              remote_fd = -1;
-        gf_fop_open_rsp_t   *rsp = NULL;
-        client_local_t      *local = NULL;
-        client_conf_t       *conf = NULL;
-        client_fd_ctx_t     *fdctx = NULL;
-
-
-        local = frame->local; frame->local = NULL;
-        conf  = frame->this->private;
-        fdctx = local->fdctx;
-
-        rsp = gf_param (hdr);
-
-        op_ret    = ntoh32 (hdr->rsp.op_ret);
-        op_errno  = ntoh32 (hdr->rsp.op_errno);
-
-        if (op_ret >= 0)
-                remote_fd = ntoh64 (rsp->fd);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "reopen on %s returned %d (%"PRId64")",
-                local->loc.path, op_ret, remote_fd);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                fdctx->remote_fd = remote_fd;
-
-                if (!fdctx->released) {
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                        fdctx = NULL;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (fdctx)
-                client_fdctx_destroy (frame->this, fdctx);
-
-        STACK_DESTROY (frame->root);
-
-        client_local_wipe (local);
-
-        return 0;
-}
-
-
-int
-protocol_client_reopen (xlator_t *this, client_fd_ctx_t *fdctx)
-{
-        int                 ret = -1;
-        gf_hdr_common_t    *hdr = NULL;
-        size_t              hdrlen = 0;
-        gf_fop_open_req_t  *req = NULL;
-        size_t              pathlen = 0;
-        client_local_t     *local = NULL;
-        inode_t            *inode = NULL;
-        char               *path = NULL;
-        call_frame_t       *frame = NULL;
-
-        inode = fdctx->inode;
-
-        ret = inode_path (inode, NULL, &path);
-        if (ret < 0) {
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local), gf_client_mt_client_local_t);
-        if (!local) {
-                goto out;
-        }
-
-        local->fdctx = fdctx;
-        local->op = protocol_client_reopen_cbk;
-        local->loc.path = path; path = NULL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                goto out;
-        }
-
-        pathlen = STRLEN_0 (local->loc.path);
-
-        hdrlen = gf_hdr_len (req, pathlen);
-        hdr    = gf_hdr_new (req, pathlen);
-
-        req    = gf_param (hdr);
-
-        req->ino   = hton64 (fdctx->ino);
-        req->gen   = hton64 (fdctx->gen);
-        req->flags = hton32 (gf_flags_from_flags (fdctx->flags));
-        req->wbflags = hton32 (fdctx->wbflags);
-        strcpy (req->path, local->loc.path);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "attempting reopen on %s", local->loc.path);
-
-        frame->local = local; local = NULL;
-
-        ret = protocol_client_xfer (frame, this,
-                                    CLIENT_CHANNEL (this, CHANNEL_LOWLAT),
-                                    GF_OP_TYPE_FOP_REQUEST, GF_PROTO_FOP_OPEN,
-                                    hdr, hdrlen, NULL, 0, NULL);
-
-        return ret;
-
-out:
-        if (frame)
-                STACK_DESTROY (frame->root);
-
-        if (local)
-                client_local_wipe (local);
-
-        if (path)
-                GF_FREE (path);
-
-        return 0;
-
-}
-
-
-int
-protocol_client_post_handshake (call_frame_t *frame, xlator_t *this)
-{
-        client_conf_t            *conf = NULL;
-        client_fd_ctx_t          *tmp = NULL;
-        client_fd_ctx_t          *fdctx = NULL;
-        xlator_list_t            *parent = NULL;
-        struct list_head          reopen_head;
-
-        conf = this->private;
-        INIT_LIST_HEAD (&reopen_head);
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                list_for_each_entry_safe (fdctx, tmp, &conf->saved_fds,
-                                          sfd_pos) {
-                        if (fdctx->remote_fd != -1)
-                                continue;
-
-                        list_del (&fdctx->sfd_pos);
-                        list_add_tail (&fdctx->sfd_pos, &reopen_head);
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        list_for_each_entry_safe (fdctx, tmp, &reopen_head, sfd_pos) {
-                list_del_init (&fdctx->sfd_pos);
-
-                if (fdctx->is_dir)
-                        protocol_client_reopendir (this, fdctx);
-                else
-                        protocol_client_reopen (this, fdctx);
-        }
-
-        parent = this->parents;
-
-        while (parent) {
-                xlator_notify (parent->xlator, GF_EVENT_CHILD_UP,
-                               this);
-                parent = parent->next;
-        }
-
-        return 0;
-}
-
-/*
- * client_setvolume_cbk - setvolume callback for client protocol
- * @frame:  call frame
- * @args: argument dictionary
- *
- * not for external reference
- */
-
-int
-client_setvolume_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                      struct iobuf *iobuf)
-{
-        client_conf_t          *conf = NULL;
-        gf_mop_setvolume_rsp_t *rsp = NULL;
-        client_connection_t    *conn = NULL;
-        glusterfs_ctx_t        *ctx = NULL;
-        xlator_t               *this = NULL;
-        xlator_list_t          *parent = NULL;
-        transport_t            *trans = NULL;
-        dict_t                 *reply = NULL;
-        char                   *remote_subvol = NULL;
-        char                   *remote_error = NULL;
-        char                   *process_uuid = NULL;
-        int32_t                 ret = -1;
-        int32_t                 op_ret   = -1;
-        int32_t                 op_errno = EINVAL;
-        int32_t                 dict_len = 0;
-        transport_t            *peer_trans = NULL;
-        uint64_t                peer_trans_int = 0;
-
-        trans = frame->local; frame->local = NULL;
-        this  = frame->this;
-        conn  = trans->xl_private;
-        conf  = this->private;
-
-        rsp = gf_param (hdr);
-
-        op_ret   = ntoh32 (hdr->rsp.op_ret);
-        op_errno = gf_error_to_errno (ntoh32 (hdr->rsp.op_errno));
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setvolume failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        reply = dict_new ();
-        GF_VALIDATE_OR_GOTO (this->name, reply, out);
-
-        dict_len = ntoh32 (rsp->dict_len);
-        ret = dict_unserialize (rsp->buf, dict_len, &reply);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "failed to unserialize buffer(%p) to dictionary",
-                        rsp->buf);
-                goto out;
-        }
-
-        ret = dict_get_str (reply, "ERROR", &remote_error);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get ERROR string from reply dictionary");
-        }
-
-        ret = dict_get_str (reply, "process-uuid", &process_uuid);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get 'process-uuid' from reply dictionary");
-        }
-
-        if (op_ret < 0) {
-                gf_log (trans->xl->name, GF_LOG_ERROR,
-                        "SETVOLUME on remote-host failed: %s",
-                        remote_error ? remote_error : strerror (op_errno));
-                errno = op_errno;
-                if (op_errno == ESTALE) {
-                        parent = trans->xl->parents;
-                        while (parent) {
-                                xlator_notify (parent->xlator,
-                                               GF_EVENT_VOLFILE_MODIFIED,
-                                               trans->xl);
-                                parent = parent->next;
-                        }
-                }
-
-        } else {
-                ret = dict_get_str (this->options, "remote-subvolume",
-                                    &remote_subvol);
-                if (!remote_subvol)
-                        goto out;
-
-                ctx = this->ctx;
-
-                if (process_uuid && !strcmp (ctx->process_uuid,process_uuid)) {
-                        ret = dict_get_uint64 (reply, "transport-ptr",
-                                               &peer_trans_int);
-
-                        peer_trans = (void *) (long) (peer_trans_int);
-
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "attaching to the local volume '%s'",
-                                remote_subvol);
-
-                        transport_setpeer (trans, peer_trans);
-
-                }
-
-                gf_log (trans->xl->name, GF_LOG_INFO,
-                        "Connected to %s, attached "
-                        "to remote volume '%s'.",
-                        trans->peerinfo.identifier, remote_subvol);
-
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        conn->connected = 1;
-                }
-                pthread_mutex_unlock (&(conn->lock));
-
-                protocol_client_post_handshake (frame, frame->this);
-        }
-
-        conf->connecting = 0;
-out:
-
-        if (-1 == op_ret) {
-                /* Let the connection/re-connection happen in
-                 * background, for now, don't hang here,
-                 * tell the parents that i am all ok..
-                 */
-                parent = trans->xl->parents;
-                while (parent) {
-                        xlator_notify (parent->xlator,
-                                       GF_EVENT_CHILD_CONNECTING, trans->xl);
-                        parent = parent->next;
-                }
-                conf->connecting= 1;
-        }
-
-        STACK_DESTROY (frame->root);
-
-        if (reply)
-                dict_unref (reply);
-
-        return op_ret;
-}
-
-/*
- * client_enosys_cbk -
- * @frame: call frame
- *
- * not for external reference
- */
-
-int
-client_enosys_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-void
-client_protocol_reconnect (void *trans_ptr)
-{
-        transport_t         *trans = NULL;
-        client_connection_t *conn = NULL;
-        struct timeval       tv = {0, 0};
-        int32_t              ret = 0;
-
-        trans = trans_ptr;
-        conn  = trans->xl_private;
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->reconnect)
-                        gf_timer_call_cancel (trans->xl->ctx,
-                                              conn->reconnect);
-                conn->reconnect = 0;
-
-                if (conn->connected == 0) {
-                        tv.tv_sec = 10;
-
-                        gf_log (trans->xl->name, GF_LOG_TRACE,
-                                "attempting reconnect");
-                        ret = transport_connect (trans);
-
-                        conn->reconnect =
-                                gf_timer_call_after (trans->xl->ctx, tv,
-                                                     client_protocol_reconnect,
-                                                     trans);
-                } else {
-                        gf_log (trans->xl->name, GF_LOG_TRACE,
-                                "breaking reconnect chain");
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (ret == -1 && errno != EINPROGRESS) {
-                default_notify (trans->xl, GF_EVENT_CHILD_DOWN, NULL);
-        }
-}
-
-int
-protocol_client_mark_fd_bad (xlator_t *this)
-{
-        client_conf_t            *conf = NULL;
-        client_fd_ctx_t          *tmp = NULL;
-        client_fd_ctx_t          *fdctx = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                list_for_each_entry_safe (fdctx, tmp, &conf->saved_fds,
-                                          sfd_pos) {
-                        fdctx->remote_fd = -1;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        return 0;
-}
-
-/*
- * client_protocol_cleanup - cleanup function
- * @trans: transport object
- *
- */
-
-int
-protocol_client_cleanup (transport_t *trans)
-{
-        client_connection_t    *conn = NULL;
-        struct saved_frames    *saved_frames = NULL;
-
-        conn = trans->xl_private;
-
-        gf_log (trans->xl->name, GF_LOG_TRACE,
-                "cleaning up state in transport object %p", trans);
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                saved_frames = conn->saved_frames;
-                conn->saved_frames = gf_client_saved_frames_new ();
-
-                /* bailout logic cleanup */
-                if (conn->timer) {
-                        gf_timer_call_cancel (trans->xl->ctx, conn->timer);
-                        conn->timer = NULL;
-                }
-
-                if (conn->reconnect == NULL) {
-                        /* :O This part is empty.. any thing missing? */
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        gf_client_saved_frames_destroy (trans->xl, saved_frames,
-                                        gf_fops, gf_mops, gf_cbks);
-
-        return 0;
-}
-
-
-/* cbk callbacks */
-int
-client_releasedir_cbk (call_frame_t *frame, gf_hdr_common_t *hdr,
-                       size_t hdrlen, struct iobuf *iobuf)
-{
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-int
-client_release_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-int
-client_forget_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_log ("", GF_LOG_CRITICAL, "fop not implemented");
-        return 0;
-}
-
-
-int
-client_log_cbk (call_frame_t *frame, gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_log ("", GF_LOG_CRITICAL, "fop not implemented");
-        return 0;
-}
-
-
-static gf_op_t gf_fops[] = {
-        [GF_PROTO_FOP_STAT]           =  client_stat_cbk,
-        [GF_PROTO_FOP_READLINK]       =  client_readlink_cbk,
-        [GF_PROTO_FOP_MKNOD]          =  client_mknod_cbk,
-        [GF_PROTO_FOP_MKDIR]          =  client_mkdir_cbk,
-        [GF_PROTO_FOP_UNLINK]         =  client_unlink_cbk,
-        [GF_PROTO_FOP_RMDIR]          =  client_rmdir_cbk,
-        [GF_PROTO_FOP_SYMLINK]        =  client_symlink_cbk,
-        [GF_PROTO_FOP_RENAME]         =  client_rename_cbk,
-        [GF_PROTO_FOP_LINK]           =  client_link_cbk,
-        [GF_PROTO_FOP_TRUNCATE]       =  client_truncate_cbk,
-        [GF_PROTO_FOP_OPEN]           =  client_open_cbk,
-        [GF_PROTO_FOP_READ]           =  client_readv_cbk,
-        [GF_PROTO_FOP_WRITE]          =  client_write_cbk,
-        [GF_PROTO_FOP_STATFS]         =  client_statfs_cbk,
-        [GF_PROTO_FOP_FLUSH]          =  client_flush_cbk,
-        [GF_PROTO_FOP_FSYNC]          =  client_fsync_cbk,
-        [GF_PROTO_FOP_SETXATTR]       =  client_setxattr_cbk,
-        [GF_PROTO_FOP_GETXATTR]       =  client_getxattr_cbk,
-        [GF_PROTO_FOP_REMOVEXATTR]    =  client_removexattr_cbk,
-        [GF_PROTO_FOP_OPENDIR]        =  client_opendir_cbk,
-        [GF_PROTO_FOP_FSYNCDIR]       =  client_fsyncdir_cbk,
-        [GF_PROTO_FOP_ACCESS]         =  client_access_cbk,
-        [GF_PROTO_FOP_CREATE]         =  client_create_cbk,
-        [GF_PROTO_FOP_FTRUNCATE]      =  client_ftruncate_cbk,
-        [GF_PROTO_FOP_FSTAT]          =  client_fstat_cbk,
-        [GF_PROTO_FOP_LK]             =  client_lk_common_cbk,
-        [GF_PROTO_FOP_LOOKUP]         =  client_lookup_cbk,
-        [GF_PROTO_FOP_READDIR]        =  client_readdir_cbk,
-        [GF_PROTO_FOP_READDIRP]       =  client_readdirp_cbk,
-        [GF_PROTO_FOP_INODELK]        =  client_inodelk_cbk,
-        [GF_PROTO_FOP_FINODELK]       =  client_finodelk_cbk,
-        [GF_PROTO_FOP_ENTRYLK]        =  client_entrylk_cbk,
-        [GF_PROTO_FOP_FENTRYLK]       =  client_fentrylk_cbk,
-        [GF_PROTO_FOP_RCHECKSUM]      =  client_rchecksum_cbk,
-        [GF_PROTO_FOP_XATTROP]        =  client_xattrop_cbk,
-        [GF_PROTO_FOP_FXATTROP]       =  client_fxattrop_cbk,
-        [GF_PROTO_FOP_SETATTR]        =  client_setattr_cbk,
-        [GF_PROTO_FOP_FSETATTR]       =  client_fsetattr_cbk
-};
-
-static gf_op_t gf_mops[] = {
-        [GF_MOP_SETVOLUME]        =  client_setvolume_cbk,
-        [GF_MOP_GETVOLUME]        =  client_enosys_cbk,
-        [GF_MOP_SETSPEC]          =  client_setspec_cbk,
-        [GF_MOP_GETSPEC]          =  client_getspec_cbk,
-        [GF_MOP_PING]             =  client_ping_cbk,
-        [GF_MOP_LOG]              =  client_log_cbk
-};
-
-static gf_op_t gf_cbks[] = {
-        [GF_CBK_FORGET]           = client_forget_cbk,
-        [GF_CBK_RELEASE]          = client_release_cbk,
-        [GF_CBK_RELEASEDIR]       = client_releasedir_cbk
-};
-
-/*
- * client_protocol_interpret - protocol interpreter
- * @trans: transport object
- * @blk: data block
- *
- */
-int
-protocol_client_interpret (xlator_t *this, transport_t *trans,
-                           char *hdr_p, size_t hdrlen, struct iobuf *iobuf)
-{
-        int                  ret = -1;
-        call_frame_t        *frame = NULL;
-        gf_hdr_common_t     *hdr = NULL;
-        uint64_t             callid = 0;
-        int                  type = -1;
-        int                  op = -1;
-        client_connection_t *conn = NULL;
-
-        conn  = trans->xl_private;
-
-        hdr  = (gf_hdr_common_t *)hdr_p;
-
-        type   = ntoh32 (hdr->type);
-        op     = ntoh32 (hdr->op);
-        callid = ntoh64 (hdr->callid);
-
-        frame  = lookup_frame (trans, op, type, callid);
-        if (frame == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "no frame for callid=%"PRId64" type=%d op=%d",
-                        callid, type, op);
-                return 0;
-        }
-
-        switch (type) {
-        case GF_OP_TYPE_FOP_REPLY:
-                if ((op > GF_PROTO_FOP_MAXVALUE) ||
-                    (op < 0)) {
-                        gf_log (trans->xl->name, GF_LOG_WARNING,
-                                "invalid fop '%d'", op);
-                } else {
-                        ret = gf_fops[op] (frame, hdr, hdrlen, iobuf);
-                }
-                break;
-        case GF_OP_TYPE_MOP_REPLY:
-                if ((op > GF_MOP_MAXVALUE) ||
-                    (op < 0)) {
-                        gf_log (trans->xl->name, GF_LOG_WARNING,
-                                "invalid fop '%d'", op);
-                } else {
-                        ret = gf_mops[op] (frame, hdr, hdrlen, iobuf);
-                }
-                break;
-        case GF_OP_TYPE_CBK_REPLY:
-                if ((op > GF_CBK_MAXVALUE) ||
-                    (op < 0)) {
-                        gf_log (trans->xl->name, GF_LOG_WARNING,
-                                "invalid cbk '%d'", op);
-                } else {
-                        ret = gf_cbks[op] (frame, hdr, hdrlen, iobuf);
-                }
-                break;
-        default:
-                gf_log (trans->xl->name, GF_LOG_DEBUG,
-                        "invalid packet type: %d", type);
-                break;
-        }
-
-        return ret;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret       = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_client_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-
-/*
- * init - initiliazation function. called during loading of client protocol
- * @this:
- *
- */
-
-int
-init (xlator_t *this)
-{
-        transport_t               *trans = NULL;
-        client_conf_t             *conf = NULL;
-        client_connection_t       *conn = NULL;
-        int32_t                    frame_timeout = 0;
-        int32_t                    ping_timeout = 0;
-        data_t                    *remote_subvolume = NULL;
-        int32_t                    ret = -1;
-        int                        i = 0;
-
-        if (this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: client protocol translator cannot have any "
-                        "subvolumes");
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling. ");
-        }
-
-        remote_subvolume = dict_get (this->options, "remote-subvolume");
-        if (remote_subvolume == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Option 'remote-subvolume' is not specified.");
-                goto out;
-        }
-
-        ret = dict_get_int32 (this->options, "frame-timeout",
-                              &frame_timeout);
-        if (ret >= 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setting frame-timeout to %d", frame_timeout);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "defaulting frame-timeout to 30mins");
-                frame_timeout = 1800;
-        }
-
-        ret = dict_get_int32 (this->options, "ping-timeout",
-                              &ping_timeout);
-        if (ret >= 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setting ping-timeout to %d", ping_timeout);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "defaulting ping-timeout to 42");
-                ping_timeout = GF_UNIVERSAL_ANSWER;
-        }
-
-        conf = GF_CALLOC (1, sizeof (client_conf_t),
-                          gf_client_mt_client_conf_t);
-
-        protocol_common_init ();
-
-        pthread_mutex_init (&conf->mutex, NULL);
-        INIT_LIST_HEAD (&conf->saved_fds);
-
-        this->private = conf;
-
-        for (i = 0; i < CHANNEL_MAX; i++) {
-                if (CHANNEL_LOWLAT == i) {
-                        dict_set (this->options, "transport.socket.lowlat",
-                                  data_from_dynstr (gf_strdup ("true")));
-                }
-                trans = transport_load (this->options, this);
-                if (trans == NULL) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to load transport");
-                        ret = -1;
-                        goto out;
-                }
-
-                conn = GF_CALLOC (1, sizeof (*conn),
-                                  gf_client_mt_client_connection_t);
-
-                conn->saved_frames = gf_client_saved_frames_new ();
-
-                conn->callid = 1;
-
-                conn->frame_timeout = frame_timeout;
-                conn->ping_timeout = ping_timeout;
-
-                pthread_mutex_init (&conn->lock, NULL);
-
-                trans->xl_private = conn;
-                conf->transport[i] = transport_ref (trans);
-        }
-
-#ifndef GF_DARWIN_HOST_OS
-        {
-                struct rlimit lim;
-
-                lim.rlim_cur = 1048576;
-                lim.rlim_max = 1048576;
-
-                ret = setrlimit (RLIMIT_NOFILE, &lim);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "WARNING: Failed to set 'ulimit -n 1M': %s",
-                                strerror(errno));
-                        lim.rlim_cur = 65536;
-                        lim.rlim_max = 65536;
-
-                        ret = setrlimit (RLIMIT_NOFILE, &lim);
-                        if (ret == -1) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Failed to set max open fd to 64k: %s",
-                                        strerror(errno));
-                        } else {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "max open fd set to 64k");
-                        }
-
-                }
-        }
-#endif
-        ret = 0;
-out:
-        return ret;
-}
-
-/*
- * fini - finish function called during unloading of client protocol
- * @this:
- *
- */
-void
-fini (xlator_t *this)
-{
-        /* TODO: Check if its enough.. how to call transport's fini () */
-        client_conf_t *conf = NULL;
-
-        conf = this->private;
-        this->private = NULL;
-
-        if (conf) {
-                GF_FREE (conf);
-        }
-        return;
-}
-
-
-int
-protocol_client_handshake (xlator_t *this, transport_t *trans)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_mop_setvolume_req_t *req = NULL;
-        dict_t                 *options = NULL;
-        int32_t                 ret = -1;
-        int                     hdrlen = 0;
-        int                     dict_len = 0;
-        call_frame_t           *fr = NULL;
-        char                   *process_uuid_xl;
-
-        options = this->options;
-        ret = dict_set_str (options, "protocol-version", GF_PROTOCOL_VERSION);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set protocol version(%s) in handshake msg",
-                        GF_PROTOCOL_VERSION);
-        }
-
-        ret = gf_asprintf (&process_uuid_xl, "%s-%s", this->ctx->process_uuid,
-                           this->name);
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "asprintf failed while setting process_uuid");
-                goto fail;
-        }
-        ret = dict_set_dynstr (options, "process-uuid",
-                               process_uuid_xl);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set process-uuid(%s) in handshake msg",
-                        process_uuid_xl);
-        }
-
-        if (this->ctx->cmd_args.volfile_server) {
-                if (this->ctx->cmd_args.volfile_id)
-                        ret = dict_set_str (options, "volfile-key",
-                                            this->ctx->cmd_args.volfile_id);
-                ret = dict_set_uint32 (options, "volfile-checksum",
-                                       this->graph->volfile_checksum);
-        }
-
-        dict_len = dict_serialized_length (options);
-        if (dict_len < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get serialized length of dict(%p)",
-                        options);
-                ret = dict_len;
-                goto fail;
-        }
-
-        hdrlen = gf_hdr_len (req, dict_len);
-        hdr    = gf_hdr_new (req, dict_len);
-        GF_VALIDATE_OR_GOTO (this->name, hdr, fail);
-
-        req    = gf_param (hdr);
-
-        ret = dict_serialize (options, req->buf);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to serialize dictionary(%p)",
-                        options);
-                goto fail;
-        }
-
-        req->dict_len = hton32 (dict_len);
-        fr  = create_frame (this, this->ctx->pool);
-        GF_VALIDATE_OR_GOTO (this->name, fr, fail);
-
-        fr->local = trans;
-        ret = protocol_client_xfer (fr, this, trans,
-                                    GF_OP_TYPE_MOP_REQUEST, GF_MOP_SETVOLUME,
-                                    hdr, hdrlen, NULL, 0, NULL);
-        return ret;
-fail:
-        if (hdr)
-                GF_FREE (hdr);
-        return ret;
-}
-
-
-int
-protocol_client_pollout (xlator_t *this, transport_t *trans)
-{
-        client_conf_t *conf = NULL;
-
-        conf = trans->xl->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                gettimeofday (&conf->last_sent, NULL);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        return 0;
-}
-
-
-int
-protocol_client_pollin (xlator_t *this, transport_t *trans)
-{
-        client_conf_t *conf = NULL;
-        int            ret = -1;
-        struct iobuf  *iobuf = NULL;
-        char          *hdr = NULL;
-        size_t         hdrlen = 0;
-
-        conf = trans->xl->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                gettimeofday (&conf->last_received, NULL);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = transport_receive (trans, &hdr, &hdrlen, &iobuf);
-
-        if (ret == 0)
-        {
-                ret = protocol_client_interpret (this, trans, hdr, hdrlen,
-                                                 iobuf);
-        }
-
-        /* TODO: use mem-pool */
-        GF_FREE (hdr);
-
-        return ret;
-}
-
-int
-client_priv_dump (xlator_t *this)
-{
-        client_conf_t   *conf = NULL;
-        int             ret   = -1;
-        client_fd_ctx_t *tmp = NULL;
-        int             i = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-
-        if (!this)
-                return -1;
-
-        conf = this->private;
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-        }
-
-        ret = pthread_mutex_trylock(&conf->mutex);
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to lock client %s"
-                       " errno: %d", this->name, errno);
-                return -1;
-        }
-
-        gf_proc_dump_build_key(key_prefix, "xlator.protocol.client",
-                               "%s.priv", this->name);
-
-        gf_proc_dump_add_section(key_prefix);
-
-        list_for_each_entry(tmp, &conf->saved_fds, sfd_pos) {
-                gf_proc_dump_build_key(key, key_prefix,
-                                       "fd.%d.remote_fd", ++i);
-                gf_proc_dump_write(key, "%d", tmp->remote_fd);
-        }
-
-        gf_proc_dump_build_key(key, key_prefix, "connecting");
-        gf_proc_dump_write(key, "%d", conf->connecting);
-        gf_proc_dump_build_key(key, key_prefix, "last_sent");
-        gf_proc_dump_write(key, "%s", ctime(&conf->last_sent.tv_sec));
-        gf_proc_dump_build_key(key, key_prefix, "last_received");
-        gf_proc_dump_write(key, "%s", ctime(&conf->last_received.tv_sec));
-
-        pthread_mutex_unlock(&conf->mutex);
-
-        return 0;
-
-}
-
-int32_t
-client_inodectx_dump (xlator_t *this, inode_t *inode)
-{
-        ino_t   par = 0;
-        int     ret = -1;
-        char    key[GF_DUMP_MAX_BUF_LEN];
-
-        if (!inode)
-                return -1;
-
-        if (!this)
-                return -1;
-
-        ret = inode_ctx_get (inode, this, &par);
-
-        if (ret != 0)
-                return ret;
-
-        gf_proc_dump_build_key(key, "xlator.protocol.client",
-                               "%s.inode.%ld.par",
-                               this->name,inode->ino);
-        gf_proc_dump_write(key, "%ld", par);
-
-        return 0;
-}
-
-/*
- * client_protocol_notify - notify function for client protocol
- * @this:
- * @trans: transport object
- * @event
- *
- */
-
-int
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int                  i          = 0;
-        int                  ret        = 0;
-        int                  child_down = 1;
-        int                  was_not_down = 0;
-        transport_t         *trans      = NULL;
-        client_connection_t *conn       = NULL;
-        client_conf_t       *conf       = NULL;
-        xlator_list_t       *parent = NULL;
-
-        conf = this->private;
-        trans = data;
-
-        switch (event) {
-        case GF_EVENT_POLLOUT:
-        {
-                ret = protocol_client_pollout (this, trans);
-
-                break;
-        }
-        case GF_EVENT_POLLIN:
-        {
-                ret = protocol_client_pollin (this, trans);
-
-                break;
-        }
-        /* no break for ret check to happen below */
-        case GF_EVENT_POLLERR:
-        {
-                ret = -1;
-                protocol_client_cleanup (trans);
-
-                if (conf->connecting == 0) {
-                        /* Let the connection/re-connection happen in
-                         * background, for now, don't hang here,
-                         * tell the parents that i am all ok..
-                         */
-                        parent = trans->xl->parents;
-                        while (parent) {
-                                parent->xlator->notify (parent->xlator,
-                                                        GF_EVENT_CHILD_CONNECTING,
-                                                        trans->xl);
-                                parent = parent->next;
-                        }
-                        conf->connecting = 1;
-                }
-
-                was_not_down = 0;
-                for (i = 0; i < CHANNEL_MAX; i++) {
-                        conn = conf->transport[i]->xl_private;
-                        if (conn->connected == 1)
-                                was_not_down = 1;
-                }
-
-                conn = trans->xl_private;
-                if (conn->connected) {
-                        conn->connected = 0;
-                        if (conn->reconnect == 0)
-                                client_protocol_reconnect (trans);
-                }
-
-                child_down = 1;
-                for (i = 0; i < CHANNEL_MAX; i++) {
-                        trans = conf->transport[i];
-                        conn = trans->xl_private;
-                        if (conn->connected == 1)
-                                child_down = 0;
-                }
-
-                if (child_down && was_not_down) {
-                        gf_log (this->name, GF_LOG_INFO, "disconnected");
-
-                        protocol_client_mark_fd_bad (this);
-
-                        parent = this->parents;
-                        while (parent) {
-                                xlator_notify (parent->xlator,
-                                               GF_EVENT_CHILD_DOWN, this);
-                                parent = parent->next;
-                        }
-                }
-        }
-        break;
-
-        case GF_EVENT_PARENT_UP:
-        {
-                client_conf_t *conf = NULL;
-                int            i = 0;
-                transport_t   *trans = NULL;
-
-                conf = this->private;
-                for (i = 0; i < CHANNEL_MAX; i++) {
-                        trans = conf->transport[i];
-                        if (!trans) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "transport init failed");
-                                return -1;
-                        }
-
-                        conn = trans->xl_private;
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "got GF_EVENT_PARENT_UP, attempting connect "
-                                "on transport");
-
-                        client_protocol_reconnect (trans);
-                }
-        }
-        break;
-
-        case GF_EVENT_CHILD_UP:
-        {
-                char *handshake = NULL;
-
-                ret = dict_get_str (this->options, "disable-handshake",
-                                    &handshake);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "got GF_EVENT_CHILD_UP");
-                if ((ret < 0) ||
-                    (strcasecmp (handshake, "on"))) {
-                        ret = protocol_client_handshake (this, trans);
-                } else {
-                        conn = trans->xl_private;
-                        conn->connected = 1;
-                        ret = default_notify (this, event, trans);
-                }
-
-                if (ret)
-                        transport_disconnect (trans);
-
-        }
-        break;
-
-        default:
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "got %d, calling default_notify ()", event);
-
-                default_notify (this, event, data);
-                break;
-        }
-
-        return ret;
-}
-
-
-struct xlator_fops fops = {
-        .stat        = client_stat,
-        .readlink    = client_readlink,
-        .mknod       = client_mknod,
-        .mkdir       = client_mkdir,
-        .unlink      = client_unlink,
-        .rmdir       = client_rmdir,
-        .symlink     = client_symlink,
-        .rename      = client_rename,
-        .link        = client_link,
-        .truncate    = client_truncate,
-        .open        = client_open,
-        .readv       = client_readv,
-        .writev      = client_writev,
-        .statfs      = client_statfs,
-        .flush       = client_flush,
-        .fsync       = client_fsync,
-        .setxattr    = client_setxattr,
-        .getxattr    = client_getxattr,
-        .fsetxattr   = client_fsetxattr,
-        .fgetxattr   = client_fgetxattr,
-        .removexattr = client_removexattr,
-        .opendir     = client_opendir,
-        .readdir     = client_readdir,
-        .readdirp    = client_readdirp,
-        .fsyncdir    = client_fsyncdir,
-        .access      = client_access,
-        .ftruncate   = client_ftruncate,
-        .fstat       = client_fstat,
-        .create      = client_create,
-        .lk          = client_lk,
-        .inodelk     = client_inodelk,
-        .finodelk    = client_finodelk,
-        .entrylk     = client_entrylk,
-        .fentrylk    = client_fentrylk,
-        .lookup      = client_lookup,
-        .rchecksum   = client_rchecksum,
-        .xattrop     = client_xattrop,
-        .fxattrop    = client_fxattrop,
-        .setattr     = client_setattr,
-        .fsetattr    = client_fsetattr,
-        .getspec   = client_getspec,
-};
-
-struct xlator_cbks cbks = {
-        .release    = client_release,
-        .releasedir = client_releasedir
-};
-
-
-struct xlator_dumpops dumpops = {
-        .priv      =  client_priv_dump,
-        .inodectx  =  client_inodectx_dump,
-};
-
-struct volume_options options[] = {
-        { .key   = {"username"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"password"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"transport-type"},
-          .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp",
-                    "tcp/client", "ib-verbs/client"},
-          .type  = GF_OPTION_TYPE_STR
-        },
-        { .key   = {"remote-host"},
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS
-        },
-        { .key   = {"remote-subvolume"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"frame-timeout"},
-          .type  = GF_OPTION_TYPE_TIME,
-          .min   = 0,
-          .max   = 86400,
-        },
-        { .key   = {"ping-timeout"},
-          .type  = GF_OPTION_TYPE_TIME,
-          .min   = 1,
-          .max   = 1013,
-        },
-        { .key   = {NULL} },
-};
diff --git a/xlators/protocol/legacy/client/src/client-protocol.h b/xlators/protocol/legacy/client/src/client-protocol.h
deleted file mode 100644
index cd856d50e..000000000
--- a/xlators/protocol/legacy/client/src/client-protocol.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CLIENT_PROTOCOL_H
-#define _CLIENT_PROTOCOL_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h>
-#include <arpa/inet.h>
-#include "inode.h"
-#include "timer.h"
-#include "byte-order.h"
-#include "saved-frames.h"
-
-#define CLIENT_PORT_CEILING        1023
-
-#define GF_CLIENT_INODE_SELF   0
-#define GF_CLIENT_INODE_PARENT 1
-
-#define CLIENT_CONF(this) ((client_conf_t *)(this->private))
-
-#define RECEIVE_TIMEOUT(_cprivate,_current)     \
-        ((_cprivate->last_received.tv_sec +     \
-          _cprivate->frame_timeout) <           \
-         _current.tv_sec)
-
-#define SEND_TIMEOUT(_cprivate,_current)        \
-        ((_cprivate->last_sent.tv_sec +         \
-          _cprivate->frame_timeout) <           \
-         _current.tv_sec)
-
-enum {
-	CHANNEL_BULK = 0,
-	CHANNEL_LOWLAT = 1,
-	CHANNEL_MAX
-};
-
-#define CLIENT_CHANNEL client_channel
-
-struct client_connection;
-typedef struct client_connection client_connection_t;
-
-#include "stack.h"
-#include "xlator.h"
-#include "transport.h"
-#include "protocol.h"
-
-typedef struct _client_fd_ctx {
-        struct list_head  sfd_pos;      /*  Stores the reference to this
-                                            fd's position in the saved_fds list.
-                                        */
-        int64_t           remote_fd;
-        inode_t          *inode;
-        uint64_t          ino;
-        uint64_t          gen;
-        char              is_dir;
-        char              released;
-        int32_t           flags;
-        int32_t           wbflags;
-} client_fd_ctx_t;
-
-struct _client_conf {
-	transport_t         *transport[CHANNEL_MAX];
-	struct list_head     saved_fds;
-	struct timeval       last_sent;
-	struct timeval       last_received;
-	pthread_mutex_t      mutex;
-        int                  connecting;
-};
-typedef struct _client_conf client_conf_t;
-
-/* This will be stored in transport_t->xl_private */
-struct client_connection {
-	pthread_mutex_t      lock;
-	uint64_t             callid;
-	struct saved_frames *saved_frames;
-	int32_t              frame_timeout;
-	int32_t              ping_started;
-	int32_t              ping_timeout;
-	int32_t              transport_activity;
-	gf_timer_t          *reconnect;
-	char                 connected;
-	uint64_t             max_block_size;
-	gf_timer_t          *timer;
-	gf_timer_t          *ping_timer;
-};
-
-typedef struct {
-	loc_t              loc;
-	loc_t              loc2;
-	fd_t              *fd;
-        gf_op_t            op;
-        client_fd_ctx_t   *fdctx;
-        uint32_t           flags;
-        uint32_t           wbflags;
-} client_local_t;
-
-
-static inline void
-gf_string_to_stat(char *string, struct iatt *stbuf)
-{
-	uint64_t dev        = 0;
-	uint64_t ino        = 0;
-	uint32_t mode       = 0;
-	uint32_t nlink      = 0;
-	uint32_t uid        = 0;
-	uint32_t gid        = 0;
-	uint64_t rdev       = 0;
-	uint64_t size       = 0;
-	uint32_t blksize    = 0;
-	uint64_t blocks     = 0;
-	uint32_t atime      = 0;
-	uint32_t atime_nsec = 0;
-	uint32_t mtime      = 0;
-	uint32_t mtime_nsec = 0;
-	uint32_t ctime      = 0;
-	uint32_t ctime_nsec = 0;
-
-	sscanf (string, GF_STAT_PRINT_FMT_STR,
-		&dev,
-		&ino,
-		&mode,
-		&nlink,
-		&uid,
-		&gid,
-		&rdev,
-		&size,
-		&blksize,
-		&blocks,
-		&atime,
-		&atime_nsec,
-		&mtime,
-		&mtime_nsec,
-		&ctime,
-		&ctime_nsec);
-
-	stbuf->ia_gen   = dev;
-	stbuf->ia_ino   = ino;
-	stbuf->ia_prot  = ia_prot_from_st_mode (mode);
-        stbuf->ia_type  = ia_type_from_st_mode (mode);
-	stbuf->ia_nlink = nlink;
-	stbuf->ia_uid   = uid;
-	stbuf->ia_gid   = gid;
-	stbuf->ia_rdev  = rdev;
-	stbuf->ia_size  = size;
-	stbuf->ia_blksize = blksize;
-	stbuf->ia_blocks  = blocks;
-
-	stbuf->ia_atime = atime;
-	stbuf->ia_mtime = mtime;
-	stbuf->ia_ctime = ctime;
-
-	stbuf->ia_atime_nsec = atime_nsec;
-	stbuf->ia_mtime_nsec = mtime_nsec;
-	stbuf->ia_ctime_nsec = ctime_nsec;
-}
-
-#endif
diff --git a/xlators/protocol/legacy/client/src/saved-frames.c b/xlators/protocol/legacy/client/src/saved-frames.c
deleted file mode 100644
index b884a9796..000000000
--- a/xlators/protocol/legacy/client/src/saved-frames.c
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#include "saved-frames.h"
-#include "common-utils.h"
-#include "protocol.h"
-#include "xlator.h"
-#include "client-mem-types.h"
-
-
-
-struct saved_frames *
-gf_client_saved_frames_new (void)
-{
-	struct saved_frames *saved_frames = NULL;
-
-	saved_frames = GF_CALLOC (sizeof (*saved_frames), 1,
-                                  gf_client_mt_saved_frames);
-	if (!saved_frames) {
-		return NULL;
-	}
-
-        gf_log ("", 1, "here");
-	INIT_LIST_HEAD (&saved_frames->fops.list);
-	INIT_LIST_HEAD (&saved_frames->mops.list);
-	INIT_LIST_HEAD (&saved_frames->cbks.list);
-
-	return saved_frames;
-}
-
-
-struct saved_frame *
-get_head_frame_for_type (struct saved_frames *frames, int8_t type)
-{
-	struct saved_frame *head_frame = NULL;
-
-	switch (type) {
-	case GF_OP_TYPE_FOP_REQUEST:
-	case GF_OP_TYPE_FOP_REPLY:
-		head_frame = &frames->fops;
-		break;
-	case GF_OP_TYPE_MOP_REQUEST:
-	case GF_OP_TYPE_MOP_REPLY:
-		head_frame = &frames->mops;
-		break;
-	case GF_OP_TYPE_CBK_REQUEST:
-	case GF_OP_TYPE_CBK_REPLY:
-		head_frame = &frames->cbks;
-		break;
-	}
-
-	return head_frame;
-}
-
-
-int
-saved_frames_put (struct saved_frames *frames, call_frame_t *frame,
-		  int32_t op, int8_t type, int64_t callid)
-{
-	struct saved_frame *saved_frame = NULL;
-	struct saved_frame *head_frame = NULL;
-
-	head_frame = get_head_frame_for_type (frames, type);
-
-	saved_frame = GF_CALLOC (sizeof (*saved_frame), 1,
-                                gf_client_mt_saved_frame);
-	if (!saved_frame) {
-		return -ENOMEM;
-	}
-
-	INIT_LIST_HEAD (&saved_frame->list);
-	saved_frame->frame  = frame;
-	saved_frame->op     = op;
-	saved_frame->type   = type;
-	saved_frame->callid = callid;
-
-	gettimeofday (&saved_frame->saved_at, NULL);
-
-	list_add_tail (&saved_frame->list, &head_frame->list);
-	frames->count++;
-
-	return 0;
-}
-
-
-call_frame_t *
-saved_frames_get (struct saved_frames *frames, int32_t op,
-		  int8_t type, int64_t callid)
-{
-	struct saved_frame *saved_frame = NULL;
-	struct saved_frame *tmp = NULL;
-	struct saved_frame *head_frame = NULL;
-	call_frame_t       *frame = NULL;
-
-	head_frame = get_head_frame_for_type (frames, type);
-
-	list_for_each_entry (tmp, &head_frame->list, list) {
-		if (tmp->callid == callid) {
-			list_del_init (&tmp->list);
-			frames->count--;
-			saved_frame = tmp;
-			break;
-		}
-	}
-
-	if (saved_frame)
-		frame = saved_frame->frame;
-
-	GF_FREE (saved_frame);
-
-	return frame;
-}
-
-struct saved_frame *
-saved_frames_get_timedout (struct saved_frames *frames, int8_t type, 
-			   uint32_t timeout, struct timeval *current)
-{
-	struct saved_frame *bailout_frame = NULL, *tmp = NULL;
-	struct saved_frame *head_frame = NULL;
-
-	head_frame = get_head_frame_for_type (frames, type);
-
-	if (!list_empty(&head_frame->list)) {
-		tmp = list_entry (head_frame->list.next, typeof (*tmp), list);
-		if ((tmp->saved_at.tv_sec + timeout) < current->tv_sec) {
-			bailout_frame = tmp;
-			list_del_init (&bailout_frame->list);
-			frames->count--;
-		}
-	}
-
-	return bailout_frame;
-}
-
-void
-saved_frames_unwind (xlator_t *this, struct saved_frames *saved_frames,
-		     struct saved_frame *head,
-		     gf_op_t gf_ops[], char *gf_op_list[])
-{
-	struct saved_frame   *trav = NULL;
-	struct saved_frame   *tmp = NULL;
-
-	gf_hdr_common_t       hdr = {0, };
-	call_frame_t         *frame = NULL;
-
-	hdr.rsp.op_ret   = hton32 (-1);
-	hdr.rsp.op_errno = hton32 (ENOTCONN);
-
-	list_for_each_entry_safe (trav, tmp, &head->list, list) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"forced unwinding frame type(%d) op(%s)",
-			trav->type, gf_op_list[trav->op]);
-
-		hdr.type = hton32 (trav->type);
-		hdr.op   = hton32 (trav->op);
-
-		frame = trav->frame;
-
-		saved_frames->count--;
-
-		gf_ops[trav->op] (frame, &hdr, sizeof (hdr), NULL);
-
-		list_del_init (&trav->list);
-		GF_FREE (trav);
-	}
-}
-
-
-void
-gf_client_saved_frames_destroy (xlator_t *this, struct saved_frames *frames,
-                                gf_op_t gf_fops[], gf_op_t gf_mops[],
-                                gf_op_t gf_cbks[])
-{
-	saved_frames_unwind (this, frames, &frames->fops, gf_fops, gf_fop_list);
-	saved_frames_unwind (this, frames, &frames->mops, gf_mops, gf_mop_list);
-	saved_frames_unwind (this, frames, &frames->cbks, gf_cbks, gf_cbk_list);
-
-	GF_FREE (frames);
-}
diff --git a/xlators/protocol/legacy/client/src/saved-frames.h b/xlators/protocol/legacy/client/src/saved-frames.h
deleted file mode 100644
index 9f7fe0a7d..000000000
--- a/xlators/protocol/legacy/client/src/saved-frames.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SAVED_FRAMES_H
-#define _SAVED_FRAMES_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdint.h>
-#include <sys/time.h>
-#include "stack.h"
-#include "list.h"
-#include "protocol.h"
-
-/* UGLY: have common typedef b/w saved-frames.c and protocol-client.c */
-typedef int32_t (*gf_op_t) (call_frame_t *frame,
-                            gf_hdr_common_t *hdr, size_t hdrlen,
-                            struct iobuf *iobuf);
-
-
-struct saved_frame {
-	union {
-		struct list_head list;
-		struct {
-			struct saved_frame *frame_next;
-			struct saved_frame *frame_prev;
-		};
-	};
-
-	struct timeval  saved_at;
-	call_frame_t   *frame;
-	int32_t         op;
-	int8_t          type;
-	uint64_t        callid;
-};
-
-
-struct saved_frames {
-	int64_t            count;
-	struct saved_frame fops;
-	struct saved_frame mops;
-	struct saved_frame cbks;
-};
-
-
-struct saved_frames *gf_client_saved_frames_new ();
-int saved_frames_put (struct saved_frames *frames, call_frame_t *frame,
-		      int32_t op, int8_t type, int64_t callid);
-call_frame_t *saved_frames_get (struct saved_frames *frames, int32_t op,
-				int8_t type, int64_t callid);
-
-struct saved_frame *
-saved_frames_get_timedout (struct saved_frames *frames, int8_t type, 
-			   uint32_t timeout, struct timeval *current);
-
-void gf_client_saved_frames_destroy (xlator_t *this, struct saved_frames *frames,
-                                     gf_op_t gf_fops[], gf_op_t gf_mops[],
-                                     gf_op_t gf_cbks[]);
-
-#endif /* _SAVED_FRAMES_H */
diff --git a/xlators/protocol/legacy/lib/src/Makefile.am b/xlators/protocol/legacy/lib/src/Makefile.am
deleted file mode 100644
index 1f0e93e30..000000000
--- a/xlators/protocol/legacy/lib/src/Makefile.am
+++ /dev/null
@@ -1,14 +0,0 @@
-lib_LTLIBRARIES = libgfproto.la
-
-libgfproto_la_CFLAGS = -fPIC  -Wall -g -shared -nostartfiles $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS)
-
-libgfproto_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -D__USE_FILE_OFFSET64 -D_GNU_SOURCE  \
-	-D$(GF_HOST_OS) -DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\"     \
-	-DTRANSPORTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/transport\"          \
-	-I$(CONTRIBDIR)/rbtree -I$(top_srcdir)/libglusterfs/src/
-
-libgfproto_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-libgfproto_la_SOURCES = transport.c protocol.c
-
-noinst_HEADERS = transport.h protocol.h
diff --git a/xlators/protocol/legacy/lib/src/protocol.c b/xlators/protocol/legacy/lib/src/protocol.c
deleted file mode 100644
index 63950f43d..000000000
--- a/xlators/protocol/legacy/lib/src/protocol.c
+++ /dev/null
@@ -1,108 +0,0 @@
-
-#include "globals.h"
-#include "compat.h"
-#include "protocol.h"
-
-char *gf_mop_list[GF_MOP_MAXVALUE];
-char *gf_cbk_list[GF_CBK_MAXVALUE];
-
-static int
-gf_dirent_nb_size (gf_dirent_t *entries)
-{
-	return (sizeof (struct gf_dirent_nb) + strlen (entries->d_name) + 1);
-}
-
-int
-gf_dirent_serialize (gf_dirent_t *entries, char *buf, size_t buf_size)
-{
-	struct gf_dirent_nb *entry_nb = NULL;
-	gf_dirent_t         *entry = NULL;
-	int                  size = 0;
-	int                  entry_size = 0;
-
-
-	list_for_each_entry (entry, &entries->list, list) {
-		entry_size = gf_dirent_nb_size (entry);
-
-		if (buf && (size + entry_size <= buf_size)) {
-			entry_nb = (void *) (buf + size);
-
-			entry_nb->d_ino  = hton64 (entry->d_ino);
-			entry_nb->d_off  = hton64 (entry->d_off);
-			entry_nb->d_len  = hton32 (entry->d_len);
-			entry_nb->d_type = hton32 (entry->d_type);
-
-                        gf_stat_from_iatt (&entry_nb->d_stat, &entry->d_stat);
-
-			strcpy (entry_nb->d_name, entry->d_name);
-		}
-		size += entry_size;
-	}
-
-	return size;
-}
-
-
-int
-gf_dirent_unserialize (gf_dirent_t *entries, const char *buf, size_t buf_size)
-{
-	struct gf_dirent_nb *entry_nb = NULL;
-	int                  remaining_size = 0;
-	int                  least_dirent_size = 0;
-	int                  count = 0;
-	gf_dirent_t         *entry = NULL;
-	int                  entry_strlen = 0;
-	int                  entry_len = 0;
-
-
-	remaining_size = buf_size;
-	least_dirent_size = (sizeof (struct gf_dirent_nb) + 2);
-
-	while (remaining_size >= least_dirent_size) {
-		entry_nb = (void *)(buf + (buf_size - remaining_size));
-
-		entry_strlen = strnlen (entry_nb->d_name, remaining_size);
-		if (entry_strlen == remaining_size) {
-			break;
-		}
-
-		entry_len = sizeof (gf_dirent_t) + entry_strlen + 1;
-		entry = GF_CALLOC (1, entry_len, gf_common_mt_gf_dirent_t);
-		if (!entry) {
-			break;
-		}
-
-		entry->d_ino  = ntoh64 (entry_nb->d_ino);
-		entry->d_off  = ntoh64 (entry_nb->d_off);
-		entry->d_len  = ntoh32 (entry_nb->d_len);
-		entry->d_type = ntoh32 (entry_nb->d_type);
-
-                gf_stat_to_iatt (&entry_nb->d_stat, &entry->d_stat);
-
-		strcpy (entry->d_name, entry_nb->d_name);
-
-		list_add_tail (&entry->list, &entries->list);
-
-		remaining_size -= (sizeof (*entry_nb) + entry_strlen + 1);
-		count++;
-	}
-
-	return count;
-}
-
-int
-protocol_common_init (void)
-{
-        gf_mop_list[GF_MOP_SETVOLUME]   = "SETVOLUME";
-        gf_mop_list[GF_MOP_GETVOLUME]   = "GETVOLUME";
-        gf_mop_list[GF_MOP_SETSPEC]     = "SETSPEC";
-        gf_mop_list[GF_MOP_GETSPEC]     = "GETSPEC";
-        gf_mop_list[GF_MOP_LOG]         = "LOG";
-        gf_mop_list[GF_MOP_PING]        = "PING";
-
-        gf_cbk_list[GF_CBK_FORGET]      = "FORGET";
-        gf_cbk_list[GF_CBK_RELEASE]     = "RELEASE";
-        gf_cbk_list[GF_CBK_RELEASEDIR]  = "RELEASEDIR";
-
-        return 0;
-}
diff --git a/xlators/protocol/legacy/lib/src/protocol.h b/xlators/protocol/legacy/lib/src/protocol.h
deleted file mode 100644
index 6c568ed0d..000000000
--- a/xlators/protocol/legacy/lib/src/protocol.h
+++ /dev/null
@@ -1,1118 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _PROTOCOL_H
-#define _PROTOCOL_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <inttypes.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
-#include <unistd.h>
-#include <fcntl.h>
-
-#include "byte-order.h"
-#include "iatt.h"
-
-/* Any changes in the protocol structure or adding new '[f,m]ops' needs to 
- * bump the protocol version by "0.1" 
- */
-
-#define GF_PROTOCOL_VERSION "3.0"
-
-extern char *gf_mop_list[];
-extern char *gf_cbk_list[];
-
-/* NOTE: add members ONLY at the end (just before _MAXVALUE) */
-typedef enum {
-        GF_PROTO_FOP_STAT,       /* 0 */
-        GF_PROTO_FOP_READLINK,   /* 1 */
-        GF_PROTO_FOP_MKNOD,      /* 2 */
-        GF_PROTO_FOP_MKDIR,
-        GF_PROTO_FOP_UNLINK,
-        GF_PROTO_FOP_RMDIR,      /* 5 */
-        GF_PROTO_FOP_SYMLINK,
-        GF_PROTO_FOP_RENAME,
-        GF_PROTO_FOP_LINK,
-        GF_PROTO_FOP_TRUNCATE,
-        GF_PROTO_FOP_OPEN,       /* 10 */
-        GF_PROTO_FOP_READ,
-        GF_PROTO_FOP_WRITE,
-        GF_PROTO_FOP_STATFS,     /* 15 */
-        GF_PROTO_FOP_FLUSH,
-        GF_PROTO_FOP_FSYNC,
-        GF_PROTO_FOP_SETXATTR,
-        GF_PROTO_FOP_GETXATTR,
-        GF_PROTO_FOP_REMOVEXATTR,/* 20 */
-        GF_PROTO_FOP_OPENDIR,
-        GF_PROTO_FOP_GETDENTS,
-        GF_PROTO_FOP_FSYNCDIR,
-        GF_PROTO_FOP_ACCESS,
-        GF_PROTO_FOP_CREATE,     /* 25 */
-        GF_PROTO_FOP_FTRUNCATE,
-        GF_PROTO_FOP_FSTAT,
-        GF_PROTO_FOP_LK,
-        GF_PROTO_FOP_LOOKUP,
-        GF_PROTO_FOP_SETDENTS,
-        GF_PROTO_FOP_READDIR,
-        GF_PROTO_FOP_INODELK,   /* 35 */
-        GF_PROTO_FOP_FINODELK,
-	GF_PROTO_FOP_ENTRYLK,
-	GF_PROTO_FOP_FENTRYLK,
-        GF_PROTO_FOP_CHECKSUM,
-        GF_PROTO_FOP_XATTROP,  /* 40 */
-        GF_PROTO_FOP_FXATTROP,
-        GF_PROTO_FOP_LOCK_NOTIFY,
-        GF_PROTO_FOP_LOCK_FNOTIFY,
-        GF_PROTO_FOP_FGETXATTR,
-        GF_PROTO_FOP_FSETXATTR, /* 45 */
-        GF_PROTO_FOP_RCHECKSUM,
-        GF_PROTO_FOP_SETATTR,
-        GF_PROTO_FOP_FSETATTR,
-        GF_PROTO_FOP_READDIRP,
-        GF_PROTO_FOP_MAXVALUE,
-} glusterfs_proto_fop_t;
-
-/* NOTE: add members ONLY at the end (just before _MAXVALUE) */
-typedef enum {
-        GF_MOP_SETVOLUME, /* 0 */
-        GF_MOP_GETVOLUME, /* 1 */
-        GF_MOP_STATS,
-        GF_MOP_SETSPEC,
-        GF_MOP_GETSPEC,
-	GF_MOP_PING,      /* 5 */
-        GF_MOP_LOG,
-        GF_MOP_NOTIFY,
-        GF_MOP_MAXVALUE,   /* 8 */
-} glusterfs_mop_t;
-
-typedef enum {
-	GF_CBK_FORGET,      /* 0 */
-	GF_CBK_RELEASE,     /* 1 */
-	GF_CBK_RELEASEDIR,  /* 2 */
-	GF_CBK_MAXVALUE     /* 3 */
-} glusterfs_cbk_t;
-
-typedef enum {
-        GF_OP_TYPE_FOP_REQUEST = 1,
-        GF_OP_TYPE_MOP_REQUEST,
-	GF_OP_TYPE_CBK_REQUEST,
-        GF_OP_TYPE_FOP_REPLY,
-        GF_OP_TYPE_MOP_REPLY,
-	GF_OP_TYPE_CBK_REPLY
-} glusterfs_op_type_t;
-
-
-struct gf_stat {
-	uint64_t ino;
-	uint64_t size;
-	uint64_t blocks;
-	uint64_t dev;
-	uint32_t rdev;
-	uint32_t mode;
-	uint32_t nlink;
-	uint32_t uid;
-	uint32_t gid;
-	uint32_t blksize;
-	uint32_t atime;
-	uint32_t atime_nsec;
-	uint32_t mtime ;
-	uint32_t mtime_nsec;
-	uint32_t ctime;
-	uint32_t ctime_nsec;
-} __attribute__((packed));
-
-
-static inline void
-gf_stat_to_stat (struct gf_stat *gf_stat, struct stat *stat)
-{
-	stat->st_dev          = ntoh64 (gf_stat->dev);
-	stat->st_ino          = ntoh64 (gf_stat->ino);
-	stat->st_mode         = ntoh32 (gf_stat->mode);
-	stat->st_nlink        = ntoh32 (gf_stat->nlink);
-	stat->st_uid          = ntoh32 (gf_stat->uid);
-	stat->st_gid          = ntoh32 (gf_stat->gid);
-	stat->st_rdev         = ntoh32 (gf_stat->rdev);
-	stat->st_size         = ntoh64 (gf_stat->size);
-	stat->st_blksize      = ntoh32 (gf_stat->blksize);
-	stat->st_blocks       = ntoh64 (gf_stat->blocks);
-	stat->st_atime        = ntoh32 (gf_stat->atime);
-	stat->st_mtime        = ntoh32 (gf_stat->mtime);
-	stat->st_ctime        = ntoh32 (gf_stat->ctime);
-        ST_ATIM_NSEC_SET(stat, ntoh32 (gf_stat->atime_nsec));
-        ST_MTIM_NSEC_SET(stat, ntoh32 (gf_stat->mtime_nsec));
-        ST_CTIM_NSEC_SET(stat, ntoh32 (gf_stat->ctime_nsec));
-}
-
-
-static inline void
-gf_stat_from_stat (struct gf_stat *gf_stat, struct stat *stat)
-{
-	gf_stat->dev         = hton64 (stat->st_dev);
-	gf_stat->ino         = hton64 (stat->st_ino);
-	gf_stat->mode        = hton32 (stat->st_mode);
-	gf_stat->nlink       = hton32 (stat->st_nlink);
-	gf_stat->uid         = hton32 (stat->st_uid);
-	gf_stat->gid         = hton32 (stat->st_gid);
-	gf_stat->rdev        = hton32 (stat->st_rdev);
-	gf_stat->size        = hton64 (stat->st_size);
-	gf_stat->blksize     = hton32 (stat->st_blksize);
-	gf_stat->blocks      = hton64 (stat->st_blocks);
-	gf_stat->atime       = hton32 (stat->st_atime);
-	gf_stat->mtime       = hton32 (stat->st_mtime);
-	gf_stat->ctime       = hton32 (stat->st_ctime);
-        gf_stat->atime_nsec  = hton32 (ST_ATIM_NSEC(stat));
-        gf_stat->mtime_nsec  = hton32 (ST_MTIM_NSEC(stat));
-        gf_stat->ctime_nsec  = hton32 (ST_CTIM_NSEC(stat));
-}
-
-
-static inline void
-gf_stat_to_iatt (struct gf_stat *gf_stat, struct iatt *iatt)
-{
-        iatt->ia_ino        = ntoh64 (gf_stat->ino);
-        iatt->ia_dev        = ntoh64 (gf_stat->dev);
-        iatt->ia_type       = ia_type_from_st_mode (ntoh32 (gf_stat->mode));
-        iatt->ia_prot       = ia_prot_from_st_mode (ntoh32 (gf_stat->mode));
-        iatt->ia_nlink      = ntoh32 (gf_stat->nlink);
-        iatt->ia_uid        = ntoh32 (gf_stat->uid);
-        iatt->ia_gid        = ntoh32 (gf_stat->gid);
-        iatt->ia_rdev       = ntoh64 (gf_stat->rdev);
-        iatt->ia_size       = ntoh64 (gf_stat->size);
-        iatt->ia_blksize    = ntoh32 (gf_stat->blksize);
-        iatt->ia_blocks     = ntoh64 (gf_stat->blocks);
-        iatt->ia_atime      = ntoh32 (gf_stat->atime);
-        iatt->ia_atime_nsec = ntoh32 (gf_stat->atime_nsec);
-        iatt->ia_mtime      = ntoh32 (gf_stat->mtime);
-        iatt->ia_mtime_nsec = ntoh32 (gf_stat->mtime_nsec);
-        iatt->ia_ctime      = ntoh32 (gf_stat->ctime);
-        iatt->ia_ctime_nsec = ntoh32 (gf_stat->ctime_nsec);
-
-        iatt->ia_gen        = ntoh64 (gf_stat->dev);
-}
-
-
-static inline void
-gf_stat_from_iatt (struct gf_stat *gf_stat, struct iatt *iatt)
-{
-        gf_stat->ino        = hton64 (iatt->ia_ino);
-        gf_stat->dev        = hton64 (iatt->ia_dev);
-        gf_stat->mode       = hton32 (st_mode_from_ia (iatt->ia_prot,
-                                                       iatt->ia_type));
-        gf_stat->nlink      = hton32 (iatt->ia_nlink);
-        gf_stat->uid        = hton32 (iatt->ia_uid);
-        gf_stat->gid        = hton32 (iatt->ia_gid);
-        gf_stat->rdev       = hton32 (iatt->ia_rdev);
-        gf_stat->size       = hton64 (iatt->ia_size);
-        gf_stat->blksize    = hton32 (iatt->ia_blksize);
-        gf_stat->blocks     = hton64 (iatt->ia_blocks);
-        gf_stat->atime      = hton32 (iatt->ia_atime);
-        gf_stat->atime_nsec = hton32 (iatt->ia_atime_nsec);
-        gf_stat->mtime      = hton32 (iatt->ia_mtime);
-        gf_stat->mtime_nsec = hton32 (iatt->ia_mtime_nsec);
-        gf_stat->ctime      = hton32 (iatt->ia_ctime);
-        gf_stat->ctime_nsec = hton32 (iatt->ia_ctime_nsec);
-
-        gf_stat->dev        = hton64 (iatt->ia_gen);
-
-}
-
-
-struct gf_statfs {
-	uint64_t bsize;
-	uint64_t frsize;
-	uint64_t blocks;
-	uint64_t bfree;
-	uint64_t bavail;
-	uint64_t files;
-	uint64_t ffree;
-	uint64_t favail;
-	uint64_t fsid;
-	uint64_t flag;
-	uint64_t namemax;
-} __attribute__((packed));
-
-
-static inline void
-gf_statfs_to_statfs (struct gf_statfs *gf_stat, struct statvfs *stat)
-{
-	stat->f_bsize   = ntoh64 (gf_stat->bsize);
-	stat->f_frsize  = ntoh64 (gf_stat->frsize);
-	stat->f_blocks  = ntoh64 (gf_stat->blocks);
-	stat->f_bfree   = ntoh64 (gf_stat->bfree);
-	stat->f_bavail  = ntoh64 (gf_stat->bavail);
-	stat->f_files   = ntoh64 (gf_stat->files);
-	stat->f_ffree   = ntoh64 (gf_stat->ffree);
-	stat->f_favail  = ntoh64 (gf_stat->favail);
-	stat->f_fsid    = ntoh64 (gf_stat->fsid);
-	stat->f_flag    = ntoh64 (gf_stat->flag);
-	stat->f_namemax = ntoh64 (gf_stat->namemax);
-}
-
-
-static inline void
-gf_statfs_from_statfs (struct gf_statfs *gf_stat, struct statvfs *stat)
-{
-	gf_stat->bsize   = hton64 (stat->f_bsize);
-	gf_stat->frsize  = hton64 (stat->f_frsize);
-	gf_stat->blocks  = hton64 (stat->f_blocks);
-	gf_stat->bfree   = hton64 (stat->f_bfree);
-	gf_stat->bavail  = hton64 (stat->f_bavail);
-	gf_stat->files   = hton64 (stat->f_files);
-	gf_stat->ffree   = hton64 (stat->f_ffree);
-	gf_stat->favail  = hton64 (stat->f_favail);
-	gf_stat->fsid    = hton64 (stat->f_fsid);
-	gf_stat->flag    = hton64 (stat->f_flag);
-	gf_stat->namemax = hton64 (stat->f_namemax);
-}
-
-
-struct gf_flock {
-	uint16_t type;
-	uint16_t whence;
-	uint64_t start;
-	uint64_t len;
-	uint32_t pid;
-} __attribute__((packed));
-
-
-static inline void
-gf_flock_to_flock (struct gf_flock *gf_flock, struct gf_flock *flock)
-{
-	flock->l_type   = ntoh16 (gf_flock->type);
-	flock->l_whence = ntoh16 (gf_flock->whence);
-	flock->l_start  = ntoh64 (gf_flock->start);
-	flock->l_len    = ntoh64 (gf_flock->len);
-	flock->l_pid    = ntoh32 (gf_flock->pid);
-}
-
-
-static inline void
-gf_flock_from_flock (struct gf_flock *gf_flock, struct gf_flock *flock)
-{
-	gf_flock->type   = hton16 (flock->l_type);
-	gf_flock->whence = hton16 (flock->l_whence);
-	gf_flock->start  = hton64 (flock->l_start);
-	gf_flock->len    = hton64 (flock->l_len);
-	gf_flock->pid    = hton32 (flock->l_pid);
-}
-
-
-struct gf_timespec {
-	uint32_t tv_sec;
-	uint32_t tv_nsec;
-} __attribute__((packed));
-
-
-static inline void
-gf_timespec_to_timespec (struct gf_timespec *gf_ts, struct timespec *ts)
-{
-
-	ts[0].tv_sec  = ntoh32 (gf_ts[0].tv_sec);
-	ts[0].tv_nsec = ntoh32 (gf_ts[0].tv_nsec);
-	ts[1].tv_sec  = ntoh32 (gf_ts[1].tv_sec);
-	ts[1].tv_nsec = ntoh32 (gf_ts[1].tv_nsec);
-}
-
-
-static inline void
-gf_timespec_from_timespec (struct gf_timespec *gf_ts, struct timespec *ts)
-{
-	gf_ts[0].tv_sec  = hton32 (ts[0].tv_sec);
-	gf_ts[0].tv_nsec = hton32 (ts[0].tv_nsec);
-	gf_ts[1].tv_sec  = hton32 (ts[1].tv_sec);
-	gf_ts[1].tv_nsec = hton32 (ts[1].tv_nsec);
-}
-
-
-#define GF_O_ACCMODE           003
-#define GF_O_RDONLY             00
-#define GF_O_WRONLY             01
-#define GF_O_RDWR               02
-#define GF_O_CREAT            0100
-#define GF_O_EXCL             0200
-#define GF_O_NOCTTY           0400
-#define GF_O_TRUNC           01000
-#define GF_O_APPEND          02000
-#define GF_O_NONBLOCK        04000
-#define GF_O_SYNC           010000
-#define GF_O_ASYNC          020000
-
-#define GF_O_DIRECT         040000
-#define GF_O_DIRECTORY     0200000
-#define GF_O_NOFOLLOW      0400000
-#define GF_O_NOATIME      01000000
-#define GF_O_CLOEXEC      02000000
-
-#define GF_O_LARGEFILE     0100000
-
-#define XLATE_BIT(from, to, bit)    do {                \
-                if (from & bit)                         \
-                        to = to | GF_##bit;             \
-        } while (0)
-
-#define UNXLATE_BIT(from, to, bit)  do {                \
-                if (from & GF_##bit)                    \
-                        to = to | bit;                  \
-        } while (0)
-
-#define XLATE_ACCESSMODE(from, to) do {                 \
-                switch (from & O_ACCMODE) {             \
-                case O_RDONLY: to |= GF_O_RDONLY;       \
-                        break;                          \
-                case O_WRONLY: to |= GF_O_WRONLY;       \
-                        break;                          \
-                case O_RDWR: to |= GF_O_RDWR;           \
-                        break;                          \
-                }                                       \
-        } while (0)
-
-#define UNXLATE_ACCESSMODE(from, to) do {               \
-                switch (from & GF_O_ACCMODE) {          \
-                case GF_O_RDONLY: to |= O_RDONLY;       \
-                        break;                          \
-                case GF_O_WRONLY: to |= O_WRONLY;       \
-                        break;                          \
-                case GF_O_RDWR: to |= O_RDWR;           \
-                        break;                          \
-                }                                       \
-        } while (0)
-
-static inline uint32_t
-gf_flags_from_flags (uint32_t flags)
-{
-        uint32_t gf_flags = 0;
-
-        XLATE_ACCESSMODE (flags, gf_flags);
-
-        XLATE_BIT (flags, gf_flags, O_CREAT);
-        XLATE_BIT (flags, gf_flags, O_EXCL);
-        XLATE_BIT (flags, gf_flags, O_NOCTTY);
-        XLATE_BIT (flags, gf_flags, O_TRUNC);
-        XLATE_BIT (flags, gf_flags, O_APPEND);
-        XLATE_BIT (flags, gf_flags, O_NONBLOCK);
-        XLATE_BIT (flags, gf_flags, O_SYNC);
-        XLATE_BIT (flags, gf_flags, O_ASYNC);
-
-        XLATE_BIT (flags, gf_flags, O_DIRECT);
-        XLATE_BIT (flags, gf_flags, O_DIRECTORY);
-        XLATE_BIT (flags, gf_flags, O_NOFOLLOW);
-#ifdef O_NOATIME
-        XLATE_BIT (flags, gf_flags, O_NOATIME);
-#endif
-#ifdef O_CLOEXEC
-        XLATE_BIT (flags, gf_flags, O_CLOEXEC);
-#endif
-        XLATE_BIT (flags, gf_flags, O_LARGEFILE);
-
-        return gf_flags;
-}
-
-static inline uint32_t
-gf_flags_to_flags (uint32_t gf_flags)
-{
-        uint32_t flags = 0;
-
-        UNXLATE_ACCESSMODE (gf_flags, flags);
-
-        UNXLATE_BIT (gf_flags, flags, O_CREAT);
-        UNXLATE_BIT (gf_flags, flags, O_EXCL);
-        UNXLATE_BIT (gf_flags, flags, O_NOCTTY);
-        UNXLATE_BIT (gf_flags, flags, O_TRUNC);
-        UNXLATE_BIT (gf_flags, flags, O_APPEND);
-        UNXLATE_BIT (gf_flags, flags, O_NONBLOCK);
-        UNXLATE_BIT (gf_flags, flags, O_SYNC);
-        UNXLATE_BIT (gf_flags, flags, O_ASYNC);
-
-        UNXLATE_BIT (gf_flags, flags, O_DIRECT);
-        UNXLATE_BIT (gf_flags, flags, O_DIRECTORY);
-        UNXLATE_BIT (gf_flags, flags, O_NOFOLLOW);
-#ifdef O_NOATIME
-        UNXLATE_BIT (gf_flags, flags, O_NOATIME);
-#endif
-#ifdef O_CLOEXEC
-        UNXLATE_BIT (gf_flags, flags, O_CLOEXEC);
-#endif
-        UNXLATE_BIT (gf_flags, flags, O_LARGEFILE);
-
-        return flags;
-}
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	char     path[0];     /* NULL terminated */
-} __attribute__((packed)) gf_fop_stat_req_t;;
-typedef struct {
-	struct gf_stat stat;
-} __attribute__((packed)) gf_fop_stat_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t size;
-	char     path[0];     /* NULL terminated */
-} __attribute__((packed)) gf_fop_readlink_req_t;
-typedef struct {
-        struct gf_stat buf;
-	char     path[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_readlink_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	uint64_t dev;
-	uint32_t mode;
-	char     path[0];     /* NULL terminated */
-	char     bname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_mknod_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_mknod_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	uint32_t mode;
-	char     path[0];     /* NULL terminated */
-	char     bname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_mkdir_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_mkdir_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	char     path[0];     /* NULL terminated */
-	char     bname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_unlink_req_t;
-typedef struct {
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_unlink_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	char     path[0];
-	char     bname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_rmdir_req_t;
-typedef struct {
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_rmdir_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	char     path[0];
-	char     bname[0];
-	char     linkname[0];
-} __attribute__((packed)) gf_fop_symlink_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-}__attribute__((packed)) gf_fop_symlink_rsp_t;
-
-
-typedef struct {
-	uint64_t   oldpar;
-        uint64_t   oldgen;
-	uint64_t   newpar;
-        uint64_t   newgen;
-	char       oldpath[0];
-	char       oldbname[0]; /* NULL terminated */
-	char       newpath[0];
-	char       newbname[0]; /* NULL terminated */
-} __attribute__((packed)) gf_fop_rename_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preoldparent;
-        struct gf_stat postoldparent;
-        struct gf_stat prenewparent;
-        struct gf_stat postnewparent;
-} __attribute__((packed)) gf_fop_rename_rsp_t;
-
-
-typedef struct {
-	uint64_t   oldino;
-        uint64_t   oldgen;
-	uint64_t   newpar;
-        uint64_t   newgen;
-	char       oldpath[0];
-	char       newpath[0];
-	char       newbname[0];
-}__attribute__((packed)) gf_fop_link_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_link_rsp_t;
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint64_t offset;
-	char     path[0];
-} __attribute__((packed)) gf_fop_truncate_req_t;
-typedef struct {
-	struct gf_stat prestat;
-        struct gf_stat poststat;
-} __attribute__((packed)) gf_fop_truncate_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t flags;
-        uint32_t wbflags;
-	char     path[0];
-} __attribute__((packed)) gf_fop_open_req_t;
-typedef struct {
-	int64_t fd;
-} __attribute__((packed)) gf_fop_open_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-	uint32_t size;
-} __attribute__((packed)) gf_fop_read_req_t;
-typedef struct {
-	struct gf_stat stat;
-	char buf[0];
-} __attribute__((packed)) gf_fop_read_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-	uint32_t size;
-} __attribute__((packed)) gf_fop_write_req_t;
-typedef struct {
-	struct gf_stat prestat;
-        struct gf_stat poststat;
-} __attribute__((packed)) gf_fop_write_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	char     path[0];
-} __attribute__((packed)) gf_fop_statfs_req_t;
-typedef struct {
-	struct gf_statfs statfs;
-} __attribute__((packed)) gf_fop_statfs_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-} __attribute__((packed)) gf_fop_flush_req_t;
-typedef struct { } __attribute__((packed)) gf_fop_flush_rsp_t;
-
-
-typedef struct fsync_req {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint32_t data;
-} __attribute__((packed)) gf_fop_fsync_req_t;
-typedef struct {
-        struct gf_stat prestat;
-        struct gf_stat poststat;
-} __attribute__((packed)) gf_fop_fsync_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t flags;
-	uint32_t dict_len;
-	char     dict[0];
-	char     path[0];
-} __attribute__((packed)) gf_fop_setxattr_req_t;
-typedef struct { } __attribute__((packed)) gf_fop_setxattr_rsp_t;
-
-
-typedef struct {
-        uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint32_t flags;
-	uint32_t dict_len;
-	char     dict[0];
-} __attribute__((packed)) gf_fop_fsetxattr_req_t;
-typedef struct { } __attribute__((packed)) gf_fop_fsetxattr_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t flags;
-	uint32_t dict_len;
-	char     dict[0];
-	char     path[0];
-} __attribute__((packed)) gf_fop_xattrop_req_t;
-
-typedef struct {
-	uint32_t dict_len;
-	char  dict[0];
-} __attribute__((packed)) gf_fop_xattrop_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint32_t flags;
-	uint32_t dict_len;
-	char     dict[0];
-} __attribute__((packed)) gf_fop_fxattrop_req_t;
-
-typedef struct {
-	uint32_t dict_len;
-	char  dict[0];
-} __attribute__((packed)) gf_fop_fxattrop_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t namelen;
-	char     path[0];
-	char     name[0];
-} __attribute__((packed)) gf_fop_getxattr_req_t;
-typedef struct {
-	uint32_t dict_len;
-	char     dict[0];
-} __attribute__((packed)) gf_fop_getxattr_rsp_t;
-
-
-typedef struct {
-        uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-        uint32_t namelen;
-	char     name[0];
-} __attribute__((packed)) gf_fop_fgetxattr_req_t;
-typedef struct {
-	uint32_t dict_len;
-	char     dict[0];
-} __attribute__((packed)) gf_fop_fgetxattr_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	char     path[0];
-	char     name[0];
-} __attribute__((packed)) gf_fop_removexattr_req_t;
-typedef struct { } __attribute__((packed)) gf_fop_removexattr_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	char     path[0];
-} __attribute__((packed)) gf_fop_opendir_req_t;
-typedef struct {
-	int64_t fd;
-} __attribute__((packed)) gf_fop_opendir_rsp_t;
-
-
-typedef struct fsyncdir_req {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	int32_t  data;
-} __attribute__((packed)) gf_fop_fsyncdir_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_fsyncdir_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-	uint32_t size;
-} __attribute__((packed)) gf_fop_readdir_req_t;
-typedef struct {
-	uint32_t size;
-	char     buf[0];
-} __attribute__((packed)) gf_fop_readdir_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-	uint32_t size;
-} __attribute__((packed)) gf_fop_readdirp_req_t;
-typedef struct {
-	uint32_t size;
-	char     buf[0];
-} __attribute__((packed)) gf_fop_readdirp_rsp_t;
-
-
-typedef struct  {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t mask;
-	char     path[0];
-} __attribute__((packed)) gf_fop_access_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_access_rsp_t;
-
-
-typedef struct {
-	uint64_t par;
-        uint64_t gen;
-	uint32_t flags;
-	uint32_t mode;
-	char     path[0];
-	char     bname[0];
-} __attribute__((packed)) gf_fop_create_req_t;
-typedef struct {
-	struct gf_stat stat;
-	uint64_t       fd;
-        struct gf_stat preparent;
-        struct gf_stat postparent;
-} __attribute__((packed)) gf_fop_create_rsp_t;
-
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint64_t offset;
-} __attribute__((packed)) gf_fop_ftruncate_req_t;
-typedef struct {
-	struct gf_stat prestat;
-        struct gf_stat poststat;
-} __attribute__((packed)) gf_fop_ftruncate_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-} __attribute__((packed)) gf_fop_fstat_req_t;
-typedef struct {
-	struct gf_stat stat;
-} __attribute__((packed)) gf_fop_fstat_rsp_t;
-
-
-typedef struct {
-	uint64_t        ino;
-        uint64_t        gen;
-	int64_t         fd;
-	uint32_t        cmd;
-	uint32_t        type;
-	struct gf_flock flock;
-} __attribute__((packed)) gf_fop_lk_req_t;
-typedef struct {
-	struct gf_flock flock;
-} __attribute__((packed)) gf_fop_lk_rsp_t;
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	uint32_t cmd;
-	uint32_t type;
-	struct gf_flock flock;
-	char     path[0];
-        char     volume[0];
-} __attribute__((packed)) gf_fop_inodelk_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_inodelk_rsp_t;
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-	uint32_t cmd;
-	uint32_t type;
-	struct gf_flock flock;
-        char volume[0];
-} __attribute__((packed)) gf_fop_finodelk_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_finodelk_rsp_t;
-
-typedef struct {
-	uint64_t  ino;
-        uint64_t  gen;
-	uint32_t  cmd;
-	uint32_t  type;
-	uint64_t  namelen;
-	char      path[0];
-	char      name[0];
-        char      volume[0];
-} __attribute__((packed)) gf_fop_entrylk_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_entrylk_rsp_t;
-
-typedef struct {
-	uint64_t  ino;
-        uint64_t  gen;
-	int64_t   fd;
-	uint32_t  cmd;
-	uint32_t  type;
-	uint64_t  namelen;
-	char      name[0];
-        char      volume[0];
-} __attribute__((packed)) gf_fop_fentrylk_req_t;
-typedef struct {
-} __attribute__((packed)) gf_fop_fentrylk_rsp_t;
-
-typedef struct {
-	uint64_t ino; /* NOTE: used only in case of 'root' lookup */
-	uint64_t par;
-        uint64_t gen;
-	uint32_t flags;
-	uint32_t dictlen;
-	char     path[0];
-	char     bname[0];
-	char     dict[0];
-} __attribute__((packed)) gf_fop_lookup_req_t;
-typedef struct {
-	struct gf_stat stat;
-        struct gf_stat postparent;
-	uint32_t       dict_len;
-	char           dict[0];
-} __attribute__((packed)) gf_fop_lookup_rsp_t;
-
-typedef struct {
-	uint64_t  ino;
-        uint64_t  gen;
-	uint32_t  flag;
-	char      path[0];
-} __attribute__((packed)) gf_fop_checksum_req_t;
-typedef struct {
-	unsigned char fchecksum[0];
-	unsigned char dchecksum[0];
-} __attribute__((packed)) gf_fop_checksum_rsp_t;
-
-typedef struct {
-        uint64_t       ino;
-        uint64_t       gen;
-        struct gf_stat stbuf;
-        int32_t        valid;
-        char           path[0];
-} __attribute__((packed)) gf_fop_setattr_req_t;
-typedef struct {
-        struct gf_stat statpre;
-        struct gf_stat statpost;
-} __attribute__((packed)) gf_fop_setattr_rsp_t;
-
-typedef struct {
-        int64_t        fd;
-        struct gf_stat stbuf;
-        int32_t        valid;
-} __attribute__((packed)) gf_fop_fsetattr_req_t;
-typedef struct {
-        struct gf_stat statpre;
-        struct gf_stat statpost;
-} __attribute__((packed)) gf_fop_fsetattr_rsp_t;
-
-typedef struct {
-        int64_t   fd;
-        uint64_t  offset;
-        uint32_t  len;
-} __attribute__((packed)) gf_fop_rchecksum_req_t;
-typedef struct {
-        uint32_t weak_checksum;
-        unsigned char strong_checksum[0];
-} __attribute__((packed)) gf_fop_rchecksum_rsp_t;
-
-typedef struct {
-	uint32_t flags;
-	uint32_t keylen;
-	char     key[0];
-} __attribute__((packed)) gf_mop_getspec_req_t;
-typedef struct {
-	char spec[0];
-} __attribute__((packed)) gf_mop_getspec_rsp_t;
-
-
-typedef struct {
-        uint32_t msglen;
-	char     msg[0];
-} __attribute__((packed)) gf_mop_log_req_t;
-typedef struct {
-} __attribute__((packed)) gf_mop_log_rsp_t;
-
-
-typedef struct {
-	uint32_t dict_len;
-	char buf[0];
-} __attribute__((packed)) gf_mop_setvolume_req_t;
-typedef struct {
-	uint32_t dict_len;
-	char buf[0];
-} __attribute__((packed)) gf_mop_setvolume_rsp_t;
-
-
-typedef struct {
-} __attribute__((packed)) gf_mop_ping_req_t;
-typedef struct {
-} __attribute__((packed)) gf_mop_ping_rsp_t;
-
-typedef struct {
-	uint32_t  flags;
-        char buf[0];
-} __attribute__((packed)) gf_mop_notify_req_t;
-typedef struct {
-	uint32_t  flags;
-        char buf[0];
-} __attribute__((packed)) gf_mop_notify_rsp_t;
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-} __attribute__((packed)) gf_cbk_releasedir_req_t;
-typedef struct {
-} __attribute__((packed)) gf_cbk_releasedir_rsp_t;
-
-
-typedef struct {
-	uint64_t ino;
-        uint64_t gen;
-	int64_t  fd;
-} __attribute__((packed)) gf_cbk_release_req_t;
-typedef struct {
-} __attribute__((packed)) gf_cbk_release_rsp_t;
-
-
-typedef struct {
-	uint32_t count;
-	uint64_t ino_array[0];
-} __attribute__((packed)) gf_cbk_forget_req_t;
-typedef struct { } __attribute__((packed)) gf_cbk_forget_rsp_t;
-
-typedef struct {
-	uint32_t pid;
-	uint32_t uid;
-	uint32_t gid;
-
-        /* Number of groups being sent through the array above. */
-        uint32_t ngrps;
-
-        /* Array of groups to which the uid belongs apart from the primary group
-         * in gid.
-         */
-        uint32_t groups[GF_REQUEST_MAXGROUPS];
-
-        uint64_t lk_owner;
-} __attribute__ ((packed)) gf_hdr_req_t;
-
-
-typedef struct {
-	uint32_t op_ret;
-	uint32_t op_errno;
-} __attribute__ ((packed)) gf_hdr_rsp_t;
-
-
-typedef struct {
-	uint64_t callid;
-	uint32_t type;
-	uint32_t op;
-	uint32_t size;
-	union {
-		gf_hdr_req_t req;
-		gf_hdr_rsp_t rsp;
-	} __attribute__ ((packed));
-} __attribute__ ((packed)) gf_hdr_common_t;
-
-
-static inline gf_hdr_common_t *
-__gf_hdr_new (int size)
-{
-	gf_hdr_common_t *hdr = NULL;
-
-	/* TODO: use mem-pool */
-	hdr = GF_CALLOC (sizeof (gf_hdr_common_t) + size, 1,
-                         gf_common_mt_gf_hdr_common_t);
-
-	if (!hdr) {
-		return NULL;
-	}
-
-	hdr->size = hton32 (size);
-
-	return hdr;
-}
-
-
-#define gf_hdr_len(type, x) (sizeof (gf_hdr_common_t) + sizeof (*type) + x)
-#define gf_hdr_new(type, x) __gf_hdr_new (sizeof (*type) + x)
-
-
-static inline void *
-gf_param (gf_hdr_common_t *hdr)
-{
-	return ((void *)hdr) + sizeof (*hdr);
-}
-
-
-struct gf_dirent_nb {
-	uint64_t       d_ino;
-	uint64_t       d_off;
-	uint32_t       d_len;
-	uint32_t       d_type;
-        struct gf_stat d_stat;
-	char           d_name[0];
-} __attribute__((packed));
-
-int
-gf_dirent_unserialize (gf_dirent_t *entries, const char *buf, size_t buf_size);
-int
-gf_dirent_serialize (gf_dirent_t *entries, char *buf, size_t buf_size);
-
-int protocol_common_init (void);
-
-#endif
diff --git a/xlators/protocol/legacy/lib/src/transport.c b/xlators/protocol/legacy/lib/src/transport.c
deleted file mode 100644
index 4396613ec..000000000
--- a/xlators/protocol/legacy/lib/src/transport.c
+++ /dev/null
@@ -1,422 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <dlfcn.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <sys/poll.h>
-#include <fnmatch.h>
-#include <stdint.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "logging.h"
-#include "transport.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "list.h"
-
-
-transport_t *
-transport_load (dict_t *options,
-		xlator_t *xl)
-{
-	struct transport *trans = NULL, *return_trans = NULL;
-	char *name = NULL;
-	void *handle = NULL;
-	char *type = NULL;
-	char str[] = "ERROR";
-	int32_t ret = -1;
-	int8_t is_tcp = 0, is_unix = 0, is_ibsdp = 0;
-	volume_opt_list_t *vol_opt = NULL;
-
-	GF_VALIDATE_OR_GOTO("transport", options, fail);
-	GF_VALIDATE_OR_GOTO("transport", xl, fail);
-  
-	trans = GF_CALLOC (1, sizeof (struct transport),
-                           gf_common_mt_transport);
-	GF_VALIDATE_OR_GOTO("transport", trans, fail);
-
-	trans->xl = xl;
-	type = str;
-
-	/* Backward compatibility */
-	ret = dict_get_str (options, "transport-type", &type);
-	if (ret < 0) {
-		ret = dict_set_str (options, "transport-type", "socket");
-		if (ret < 0)
-			gf_log ("dict", GF_LOG_DEBUG,
-				"setting transport-type failed");
-		gf_log ("transport", GF_LOG_WARNING,
-			"missing 'option transport-type'. defaulting to "
-			"\"socket\"");
-	} else {
-		{
-			/* Backword compatibility to handle * /client,
-			 * * /server. 
-			 */
-			char *tmp = strchr (type, '/');
-			if (tmp)
-				*tmp = '\0';
-		}
-		
-		is_tcp = strcmp (type, "tcp");
-		is_unix = strcmp (type, "unix");
-		is_ibsdp = strcmp (type, "ib-sdp");
-		if ((is_tcp == 0) ||
-		    (is_unix == 0) ||
-		    (is_ibsdp == 0)) {
-			if (is_unix == 0)
-				ret = dict_set_str (options, 
-						    "transport.address-family",
-						    "unix");
-			if (is_ibsdp == 0)
-				ret = dict_set_str (options, 
-						    "transport.address-family",
-						    "inet-sdp");
-
-			if (ret < 0)
-				gf_log ("dict", GF_LOG_DEBUG,
-					"setting address-family failed");
-
-			ret = dict_set_str (options, 
-					    "transport-type", "socket");
-			if (ret < 0)
-				gf_log ("dict", GF_LOG_DEBUG,
-					"setting transport-type failed");
-		}
-	}
-
-	ret = dict_get_str (options, "transport-type", &type);
-	if (ret < 0) {
-		GF_FREE (trans);
-		gf_log ("transport", GF_LOG_ERROR,
-			"'option transport-type <xx>' missing in volume '%s'",
-			xl->name);
-		goto fail;
-	}
-
-	ret = gf_asprintf (&name, "%s/%s.so", TRANSPORTDIR, type);
-        if (-1 == ret) {
-                gf_log ("transport", GF_LOG_ERROR, "asprintf failed");
-                goto fail;
-        }
-	gf_log ("transport", GF_LOG_DEBUG,
-		"attempt to load file %s", name);
-
-	handle = dlopen (name, RTLD_NOW|RTLD_GLOBAL);
-	if (handle == NULL) {
-		gf_log ("transport", GF_LOG_ERROR, "%s", dlerror ());
-		gf_log ("transport", GF_LOG_ERROR,
-			"volume '%s': transport-type '%s' is not valid or "
-			"not found on this machine", 
-			xl->name, type);
-		GF_FREE (name);
-		GF_FREE (trans);
-		goto fail;
-	}
-	GF_FREE (name);
-	
-	trans->ops = dlsym (handle, "tops");
-	if (trans->ops == NULL) {
-		gf_log ("transport", GF_LOG_ERROR,
-			"dlsym (transport_ops) on %s", dlerror ());
-		GF_FREE (trans);
-		goto fail;
-	}
-
-	trans->init = dlsym (handle, "init");
-	if (trans->init == NULL) {
-		gf_log ("transport", GF_LOG_ERROR,
-			"dlsym (gf_transport_init) on %s", dlerror ());
-		GF_FREE (trans);
-		goto fail;
-	}
-
-	trans->fini = dlsym (handle, "fini");
-	if (trans->fini == NULL) {
-		gf_log ("transport", GF_LOG_ERROR,
-			"dlsym (gf_transport_fini) on %s", dlerror ());
-		GF_FREE (trans);
-		goto fail;
-	}
-	
-	vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
-                             gf_common_mt_volume_opt_list_t);
-	vol_opt->given_opt = dlsym (handle, "options");
-	if (vol_opt->given_opt == NULL) {
-		gf_log ("transport", GF_LOG_DEBUG,
-			"volume option validation not specified");
-	} else {
-		list_add_tail (&vol_opt->list, &xl->volume_options);
-		if (-1 == 
-		    validate_xlator_volume_options (xl, 
-						    vol_opt->given_opt)) {
-			gf_log ("transport", GF_LOG_ERROR,
-				"volume option validation failed");
-			GF_FREE (trans);
-			goto fail;
-		}
-	}
-	
-	ret = trans->init (trans);
-	if (ret != 0) {
-		gf_log ("transport", GF_LOG_ERROR,
-			"'%s' initialization failed", type);
-		GF_FREE (trans);
-		goto fail;
-	}
-
-	pthread_mutex_init (&trans->lock, NULL);
-	return_trans = trans;
-fail:
-	return return_trans;
-}
-
-
-int32_t 
-transport_submit (transport_t *this, char *buf, int32_t len,
-		  struct iovec *vector, int count,
-                  struct iobref *iobref)
-{
-	int32_t               ret = -1;
-        transport_t          *peer_trans = NULL;
-        struct iobuf         *iobuf = NULL;
-        struct transport_msg *msg = NULL;
-
-        if (this->peer_trans) {
-                peer_trans = this->peer_trans;
-
-                msg = GF_CALLOC (1, sizeof (*msg),
-                                gf_common_mt_transport_msg);
-                if (!msg) {
-                        return -ENOMEM;
-                }
-
-                msg->hdr = buf;
-                msg->hdrlen = len;
-
-                if (vector) {
-                        iobuf = iobuf_get (this->xl->ctx->iobuf_pool);
-                        if (!iobuf) {
-                                GF_FREE (msg->hdr);
-                                GF_FREE (msg);
-                                return -ENOMEM;
-                        }
-
-                        iov_unload (iobuf->ptr, vector, count);
-                        msg->iobuf = iobuf;
-                }
-
-                pthread_mutex_lock (&peer_trans->handover.mutex);
-                {
-                        list_add_tail (&msg->list, &peer_trans->handover.msgs);
-                        pthread_cond_broadcast (&peer_trans->handover.cond);
-                }
-                pthread_mutex_unlock (&peer_trans->handover.mutex);
-
-                return 0;
-        }
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-	GF_VALIDATE_OR_GOTO("transport", this->ops, fail);
-	
-	ret = this->ops->submit (this, buf, len, vector, count, iobref);
-fail:
-	return ret;
-}
-
-
-int32_t 
-transport_connect (transport_t *this)
-{
-	int ret = -1;
-	
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	ret = this->ops->connect (this);
-fail:
-	return ret;
-}
-
-
-int32_t
-transport_listen (transport_t *this)
-{
-	int ret = -1;
-	
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	ret = this->ops->listen (this);
-fail:
-	return ret;
-}
-
-
-int32_t 
-transport_disconnect (transport_t *this)
-{
-	int32_t ret = -1;
-	
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	ret = this->ops->disconnect (this);
-fail:
-	return ret;
-}
-
-
-int32_t 
-transport_destroy (transport_t *this)
-{
-	int32_t ret = -1;
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	if (this->fini)
-		this->fini (this);
-
-	pthread_mutex_destroy (&this->lock);
-	GF_FREE (this);
-fail:
-	return ret;
-}
-
-
-transport_t *
-transport_ref (transport_t *this)
-{
-	transport_t *return_this = NULL;
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-	
-	pthread_mutex_lock (&this->lock);
-	{
-		this->refcount ++;
-	}
-	pthread_mutex_unlock (&this->lock);
-	
-	return_this = this;
-fail:
-	return return_this;
-}
-
-
-int32_t
-transport_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-		   struct iobuf **iobuf_p)
-{
-	int32_t ret = -1;
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-
-        if (this->peer_trans) {
-                *hdr_p = this->handover.msg->hdr;
-                *hdrlen_p = this->handover.msg->hdrlen;
-                *iobuf_p = this->handover.msg->iobuf;
-
-                return 0;
-        }
-
-	ret = this->ops->receive (this, hdr_p, hdrlen_p, iobuf_p);
-fail:
-	return ret;
-}
-
-
-int32_t
-transport_unref (transport_t *this)
-{
-	int32_t refcount = 0;
-	int32_t ret = -1;
-
-	GF_VALIDATE_OR_GOTO("transport", this, fail);
-  
-	pthread_mutex_lock (&this->lock);
-	{
-		refcount = --this->refcount;
-	}
-	pthread_mutex_unlock (&this->lock);
-
-	if (refcount == 0) {
-		xlator_notify (this->xl, GF_EVENT_TRANSPORT_CLEANUP, this);
-		transport_destroy (this);
-	}
-	
-	ret = 0;
-fail:
-	return ret;
-}
-
-
-void *
-transport_peerproc (void *trans_data)
-{
-        transport_t          *trans = NULL;
-        struct transport_msg *msg = NULL;
-
-        trans = trans_data;
-
-        while (1) {
-                pthread_mutex_lock (&trans->handover.mutex);
-                {
-                        while (list_empty (&trans->handover.msgs))
-                                pthread_cond_wait (&trans->handover.cond,
-                                                   &trans->handover.mutex);
-
-                        msg = list_entry (trans->handover.msgs.next,
-                                          struct transport_msg, list);
-
-                        list_del_init (&msg->list);
-                }
-                pthread_mutex_unlock (&trans->handover.mutex);
-
-                trans->handover.msg = msg;
-
-                xlator_notify (trans->xl, GF_EVENT_POLLIN, trans);
-
-                GF_FREE (msg);
-        }
-}
-
-
-int
-transport_setpeer (transport_t *trans, transport_t *peer_trans)
-{
-        trans->peer_trans = transport_ref (peer_trans);
-
-        INIT_LIST_HEAD (&trans->handover.msgs);
-        pthread_cond_init (&trans->handover.cond, NULL);
-        pthread_mutex_init (&trans->handover.mutex, NULL);
-        pthread_create (&trans->handover.thread, NULL,
-                        transport_peerproc, trans);
-
-        peer_trans->peer_trans = transport_ref (trans);
-
-        INIT_LIST_HEAD (&peer_trans->handover.msgs);
-        pthread_cond_init (&peer_trans->handover.cond, NULL);
-        pthread_mutex_init (&peer_trans->handover.mutex, NULL);
-        pthread_create (&peer_trans->handover.thread, NULL,
-                        transport_peerproc, peer_trans);
-
-        return 0;
-}
diff --git a/xlators/protocol/legacy/lib/src/transport.h b/xlators/protocol/legacy/lib/src/transport.h
deleted file mode 100644
index 5b8b62caf..000000000
--- a/xlators/protocol/legacy/lib/src/transport.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __TRANSPORT_H__
-#define __TRANSPORT_H__
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <inttypes.h>
-
-struct transport_ops;
-typedef struct transport transport_t;
-
-#include "xlator.h"
-#include "dict.h"
-#include "compat.h"
-
-typedef struct peer_info {
-	struct sockaddr_storage sockaddr;
-	socklen_t sockaddr_len;
-	char identifier[UNIX_PATH_MAX];
-}peer_info_t;
-
-struct transport_msg {
-        struct list_head  list;
-        char             *hdr;
-        int               hdrlen;
-        struct iobuf     *iobuf;
-};
-
-struct transport {
-	struct transport_ops  *ops;
-	void                  *private;
-	void                  *xl_private;
-	pthread_mutex_t        lock;
-	int32_t                refcount;
-
-	xlator_t              *xl;
-	void                  *dnscache;
-	data_t                *buf;
-	int32_t              (*init)   (transport_t *this);
-	void                 (*fini)   (transport_t *this);
-	/*  int                  (*notify) (transport_t *this, int event, void *data); */
-	peer_info_t     peerinfo;
-	peer_info_t     myinfo;
-
-        transport_t    *peer_trans;
-        struct {
-                pthread_mutex_t       mutex;
-                pthread_cond_t        cond;
-                pthread_t             thread;
-                struct list_head      msgs;
-                struct transport_msg *msg;
-        } handover;
-                
-};
-
-struct transport_ops {
-	int32_t (*receive)    (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                               struct iobuf **iobuf_p);
-	int32_t (*submit)     (transport_t *this, char *buf, int len,
-                               struct iovec *vector, int count,
-                               struct iobref *iobref);
-	int32_t (*connect)    (transport_t *this);
-	int32_t (*listen)     (transport_t *this);
-	int32_t (*disconnect) (transport_t *this);
-};
-
-
-int32_t transport_listen     (transport_t *this);
-int32_t transport_connect    (transport_t *this);
-int32_t transport_disconnect (transport_t *this);
-int32_t transport_notify     (transport_t *this, int event);
-int32_t transport_submit     (transport_t *this, char *buf, int len,
-                              struct iovec *vector, int count,
-                              struct iobref *iobref);
-int32_t transport_receive    (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                              struct iobuf **iobuf_p);
-int32_t transport_destroy    (transport_t *this);
-
-transport_t *transport_load  (dict_t *options, xlator_t *xl);
-transport_t *transport_ref   (transport_t *trans);
-int32_t      transport_unref (transport_t *trans);
-
-int transport_setpeer (transport_t *trans, transport_t *trans_peer);
-
-#endif /* __TRANSPORT_H__ */
diff --git a/xlators/protocol/legacy/server/src/Makefile.am b/xlators/protocol/legacy/server/src/Makefile.am
deleted file mode 100644
index 262fec9ea..000000000
--- a/xlators/protocol/legacy/server/src/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-
-xlator_LTLIBRARIES = server-old.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/legacy/protocol
-
-server_old_la_LDFLAGS = -module -avoidversion
-
-server_old_la_SOURCES = server-protocol.c server-resolve.c server-helpers.c \
-			authenticate.c
-
-server_old_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la   \
-	$(top_builddir)/xlators/protocol/legacy/lib/src/libgfproto.la
-
-noinst_HEADERS = server-protocol.h server-helpers.h server-mem-types.h \
-	authenticate.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
-	-I$(top_srcdir)/contrib/md5/ \
-	-DDATADIR=\"$(localstatedir)\" -DCONFDIR=\"$(sysconfdir)/glusterfs\" \
-	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
-	$(GF_CFLAGS) -I$(top_srcdir)/xlators/protocol/legacy/lib/src \
-	-I$(top_srcdir)/xlators/protocol/lib/src
-
-CLEANFILES =
-
-install-data-hook:
-	ln -sf server-old.so $(DESTDIR)$(xlatordir)/server.so
diff --git a/xlators/protocol/legacy/server/src/authenticate.c b/xlators/protocol/legacy/server/src/authenticate.c
deleted file mode 100644
index a01d7003c..000000000
--- a/xlators/protocol/legacy/server/src/authenticate.c
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include <stdio.h>
-#include <dlfcn.h>
-#include <errno.h>
-#include "authenticate.h"
-
-static void
-init (dict_t *this,
-      char *key,
-      data_t *value,
-      void *data)
-{
-	void *handle = NULL;
-	char *auth_file = NULL;
-	auth_handle_t *auth_handle = NULL;
-	auth_fn_t authenticate = NULL;
-	int *error = NULL;
-        int  ret = 0;
-
-	/* It gets over written */
-	error = data;
-
-	if (!strncasecmp (key, "ip", strlen ("ip"))) {
-		gf_log ("authenticate", GF_LOG_ERROR,
-			"AUTHENTICATION MODULE \"IP\" HAS BEEN REPLACED "
-			"BY \"ADDR\"");
-		dict_set (this, key, data_from_dynptr (NULL, 0));
-		/* TODO: 1.3.x backword compatibility */
-		// *error = -1;
-		// return;
-		key = "addr";
-	}
-
-	ret = gf_asprintf (&auth_file, "%s/%s.so", LIBDIR, key);
-        if (-1 == ret) {
-                gf_log ("authenticate", GF_LOG_ERROR, "asprintf failed");
-                dict_set (this, key, data_from_dynptr (NULL, 0));
-                *error = -1;
-                return;
-        }
-
-	handle = dlopen (auth_file, RTLD_LAZY);
-	if (!handle) {
-		gf_log ("authenticate", GF_LOG_ERROR, "dlopen(%s): %s\n",
-			auth_file, dlerror ());
-		dict_set (this, key, data_from_dynptr (NULL, 0));
-		GF_FREE (auth_file);
-		*error = -1;
-		return;
-	}
-	GF_FREE (auth_file);
-
-	authenticate = dlsym (handle, "gf_auth");
-	if (!authenticate) {
-		gf_log ("authenticate", GF_LOG_ERROR,
-			"dlsym(gf_auth) on %s\n", dlerror ());
-		dict_set (this, key, data_from_dynptr (NULL, 0));
-		*error = -1;
-		return;
-	}
-
-	auth_handle = GF_CALLOC (1, sizeof (*auth_handle),
-                                 gf_common_mt_auth_handle_t);
-	if (!auth_handle) {
-		gf_log ("authenticate", GF_LOG_ERROR, "Out of memory");
-		dict_set (this, key, data_from_dynptr (NULL, 0));
-		*error = -1;
-		return;
-	}
-	auth_handle->vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
-                                       gf_common_mt_volume_opt_list_t);
-	auth_handle->vol_opt->given_opt = dlsym (handle, "options");
-	if (auth_handle->vol_opt->given_opt == NULL) {
-		gf_log ("authenticate", GF_LOG_DEBUG,
-			"volume option validation not specified");
-	}
-
-	auth_handle->authenticate = authenticate;
-	auth_handle->handle = handle;
-
-	dict_set (this, key,
-		  data_from_dynptr (auth_handle, sizeof (*auth_handle)));
-}
-
-static void
-fini (dict_t *this,
-      char *key,
-      data_t *value,
-      void *data)
-{
-	auth_handle_t *handle = data_to_ptr (value);
-	if (handle) {
-		dlclose (handle->handle);
-	}
-}
-
-int32_t
-gf_auth_init (xlator_t *xl, dict_t *auth_modules)
-{
-	int ret = 0;
-	auth_handle_t *handle = NULL;
-	data_pair_t *pair = NULL;
-	dict_foreach (auth_modules, init, &ret);
-	if (!ret) {
-		pair = auth_modules->members_list;
-		while (pair) {
-			handle = data_to_ptr (pair->value);
-			if (handle) {
-				list_add_tail (&(handle->vol_opt->list),
-					       &(xl->volume_options));
-				if (-1 ==
-				    validate_xlator_volume_options (xl,
-								    handle->vol_opt->given_opt)) {
-					gf_log ("authenticate", GF_LOG_ERROR,
-						"volume option validation "
-						"failed");
-					ret = -1;
-				}
-			}
-			pair = pair->next;
-		}
-	}
-	if (ret) {
-		gf_log (xl->name, GF_LOG_ERROR, "authentication init failed");
-		dict_foreach (auth_modules, fini, &ret);
-		ret = -1;
-	}
-	return ret;
-}
-
-static dict_t *__input_params;
-static dict_t *__config_params;
-
-void
-map (dict_t *this,
-     char *key,
-     data_t *value,
-     void *data)
-{
-	dict_t *res = data;
-	auth_fn_t authenticate;
-	auth_handle_t *handle = NULL;
-
-	if (value && (handle = data_to_ptr (value)) &&
-	    (authenticate = handle->authenticate)) {
-		dict_set (res, key,
-			  int_to_data (authenticate (__input_params,
-						     __config_params)));
-	} else {
-		dict_set (res, key, int_to_data (AUTH_DONT_CARE));
-	}
-}
-
-void
-reduce (dict_t *this,
-	char *key,
-	data_t *value,
-	void *data)
-{
-	int64_t val = 0;
-	int64_t *res = data;
-	if (!data)
-		return;
-
-	val = data_to_int64 (value);
-	switch (val)
-	{
-	case AUTH_ACCEPT:
-		if (AUTH_DONT_CARE == *res)
-			*res = AUTH_ACCEPT;
-		break;
-
-	case AUTH_REJECT:
-		*res = AUTH_REJECT;
-		break;
-
-	case AUTH_DONT_CARE:
-		break;
-	}
-}
-
-
-auth_result_t
-gf_authenticate (dict_t *input_params,
-		 dict_t *config_params,
-		 dict_t *auth_modules)
-{
-	dict_t *results = NULL;
-	int64_t result = AUTH_DONT_CARE;
-
-	results = get_new_dict ();
-	__input_params = input_params;
-	__config_params = config_params;
-
-	dict_foreach (auth_modules, map, results);
-
-	dict_foreach (results, reduce, &result);
-	if (AUTH_DONT_CARE == result) {
-		data_t *peerinfo_data = dict_get (input_params, "peer-info-name");
-		char *name = NULL;
-
-		if (peerinfo_data) {
-			name = peerinfo_data->data;
-		}
-
-		gf_log ("auth", GF_LOG_ERROR,
-			"no authentication module is interested in "
-			"accepting remote-client %s", name);
-		result = AUTH_REJECT;
-	}
-
-	dict_destroy (results);
-	return result;
-}
-
-void
-gf_auth_fini (dict_t *auth_modules)
-{
-	int32_t dummy;
-
-	dict_foreach (auth_modules, fini, &dummy);
-}
diff --git a/xlators/protocol/legacy/server/src/authenticate.h b/xlators/protocol/legacy/server/src/authenticate.h
deleted file mode 100644
index d35304db2..000000000
--- a/xlators/protocol/legacy/server/src/authenticate.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _AUTHENTICATE_H
-#define _AUTHENTICATE_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include <stdio.h>
-#include <fnmatch.h>
-#include "dict.h"
-#include "compat.h"
-#include "list.h"
-#include "xlator.h"
-
-typedef enum {
-	AUTH_ACCEPT,
-	AUTH_REJECT,
-	AUTH_DONT_CARE
-} auth_result_t;
-
-typedef auth_result_t (*auth_fn_t) (dict_t *input_params,
-				    dict_t *config_params);
-
-typedef struct {
-	void              *handle;
-	auth_fn_t          authenticate;
-	volume_opt_list_t *vol_opt;
-} auth_handle_t;
-
-auth_result_t gf_authenticate (dict_t *input_params,
-			       dict_t *config_params,
-			       dict_t *auth_modules);
-int32_t gf_auth_init (xlator_t *xl, dict_t *auth_modules);
-void gf_auth_fini (dict_t *auth_modules);
-
-#endif /* _AUTHENTICATE_H */
diff --git a/xlators/protocol/legacy/server/src/server-helpers.c b/xlators/protocol/legacy/server/src/server-helpers.c
deleted file mode 100644
index 15e46b3d6..000000000
--- a/xlators/protocol/legacy/server/src/server-helpers.c
+++ /dev/null
@@ -1,622 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "server-protocol.h"
-#include "server-helpers.h"
-
-
-
-void
-old_server_loc_wipe (loc_t *loc)
-{
-        if (loc->parent) {
-                inode_unref (loc->parent);
-                loc->parent = NULL;
-        }
-
-        if (loc->inode) {
-                inode_unref (loc->inode);
-                loc->inode = NULL;
-        }
-
-	if (loc->path)
-		GF_FREE ((char *)loc->path);
-}
-
-
-static void
-old_server_resolve_wipe (server_resolve_t *resolve)
-{
-        struct resolve_comp *comp = NULL;
-        int                  i = 0;
-
-        if (resolve->path)
-                GF_FREE (resolve->path);
-
-        if (resolve->bname)
-                GF_FREE (resolve->bname);
-
-        if (resolve->resolved)
-                GF_FREE (resolve->resolved);
-
-        loc_wipe (&resolve->deep_loc);
-
-        comp = resolve->components;
-        if (comp) {
-                for (i = 0; comp[i].basename; i++) {
-                        if (comp[i].inode)
-                                inode_unref (comp[i].inode);
-                }
-                GF_FREE (resolve->components);
-        }
-}
-
-
-void
-free_old_server_state (server_state_t *state)
-{
-        if (state->trans) {
-                transport_unref (state->trans);
-                state->trans = NULL;
-        }
-
-        if (state->fd) {
-                fd_unref (state->fd);
-                state->fd = NULL;
-        }
-
-        if (state->params) {
-                dict_unref (state->params);
-                state->params = NULL;
-        }
-
-        if (state->iobref) {
-                iobref_unref (state->iobref);
-                state->iobref = NULL;
-        }
-
-        if (state->iobuf) {
-                iobuf_unref (state->iobuf);
-                state->iobuf = NULL;
-        }
-
-        if (state->dict) {
-                dict_unref (state->dict);
-                state->dict = NULL;
-        }
-
-        if (state->volume)
-                GF_FREE ((char *)state->volume);
-
-        if (state->name)
-                GF_FREE (state->name);
-
-        old_server_loc_wipe (&state->loc);
-        old_server_loc_wipe (&state->loc2);
-
-        old_server_resolve_wipe (&state->resolve);
-        old_server_resolve_wipe (&state->resolve2);
-
-	GF_FREE (state);
-}
-
-static struct _lock_table *
-gf_lock_table_new (void)
-{
-        struct _lock_table *new = NULL;
-
-	new = GF_CALLOC (1, sizeof (struct _lock_table),
-                         gf_server_mt_lock_table);
-        if (new == NULL) {
-                gf_log ("server-protocol", GF_LOG_CRITICAL,
-                        "failed to allocate memory for new lock table");
-                goto out;
-        }
-        INIT_LIST_HEAD (&new->dir_lockers);
-        INIT_LIST_HEAD (&new->file_lockers);
-        LOCK_INIT (&new->lock);
-out:
-        return new;
-}
-
-
-static int
-gf_server_nop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (state)
-                free_old_server_state (state);
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-static int
-do_lock_table_cleanup (xlator_t *this, server_connection_t *conn,
-                       call_frame_t *frame, struct _lock_table *ltable)
-{
-        struct list_head  file_lockers, dir_lockers;
-        call_frame_t     *tmp_frame = NULL;
-        struct gf_flock      flock = {0, };
-        xlator_t         *bound_xl = NULL;
-        struct _locker   *locker = NULL, *tmp = NULL;
-        int               ret = -1;
-
-        bound_xl = conn->bound_xl;
-        INIT_LIST_HEAD (&file_lockers);
-        INIT_LIST_HEAD (&dir_lockers);
-
-        LOCK (&ltable->lock);
-        {
-                list_splice_init (&ltable->file_lockers,
-                                  &file_lockers);
-
-                list_splice_init (&ltable->dir_lockers, &dir_lockers);
-        }
-        UNLOCK (&ltable->lock);
-
-        GF_FREE (ltable);
-
-        flock.l_type  = F_UNLCK;
-        flock.l_start = 0;
-        flock.l_len   = 0;
-        list_for_each_entry_safe (locker,
-                                  tmp, &file_lockers, lockers) {
-                tmp_frame = copy_frame (frame);
-                if (tmp_frame == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "out of memory");
-                        goto out;
-                }
-                /*
-                   pid = 0 is a special case that tells posix-locks
-                   to release all locks from this transport
-                */
-                tmp_frame->root->pid = 0;
-                tmp_frame->root->trans = conn;
-
-                if (locker->fd) {
-                        STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                    bound_xl,
-                                    bound_xl->fops->finodelk,
-                                    locker->volume,
-                                    locker->fd, F_SETLK, &flock);
-                        fd_unref (locker->fd);
-                } else {
-                        STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                    bound_xl,
-                                    bound_xl->fops->inodelk,
-                                    locker->volume,
-                                    &(locker->loc), F_SETLK, &flock);
-                        loc_wipe (&locker->loc);
-                }
-
-                GF_FREE (locker->volume);
-                
-                list_del_init (&locker->lockers);
-                GF_FREE (locker);
-        }
-
-        tmp = NULL;
-        locker = NULL;
-        list_for_each_entry_safe (locker, tmp, &dir_lockers, lockers) {
-                tmp_frame = copy_frame (frame);
-
-                tmp_frame->root->pid = 0;
-                tmp_frame->root->trans = conn;
-
-                if (locker->fd) {
-                        STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                    bound_xl,
-                                    bound_xl->fops->fentrylk,
-                                    locker->volume,
-                                    locker->fd, NULL,
-                                    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                        fd_unref (locker->fd);
-                } else {
-                        STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                    bound_xl,
-                                    bound_xl->fops->entrylk,
-                                    locker->volume,
-                                    &(locker->loc), NULL,
-                                    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                        loc_wipe (&locker->loc);
-                }
-
-                GF_FREE (locker->volume);
-                
-                list_del_init (&locker->lockers);
-                GF_FREE (locker);
-        }
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
-static int
-server_connection_cleanup_flush_cbk (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int32_t op_ret,
-                                     int32_t op_errno)
-{
-        fd_t *fd = NULL;
-
-        fd = frame->local;
-
-        fd_unref (fd);
-        frame->local = NULL;
-
-        STACK_DESTROY (frame->root);
-        return 0;
-}
-
-
-static int
-do_fd_cleanup (xlator_t *this, server_connection_t *conn, call_frame_t *frame,
-               fdentry_t *fdentries, int fd_count)
-{
-        fd_t               *fd = NULL;
-        int                 i = 0, ret = -1;
-        call_frame_t       *tmp_frame = NULL;
-        xlator_t           *bound_xl = NULL;
-
-        bound_xl = conn->bound_xl;
-        for (i = 0;i < fd_count; i++) {
-                fd = fdentries[i].fd;
-
-                if (fd != NULL) {
-                        tmp_frame = copy_frame (frame);
-                        if (tmp_frame == NULL) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "out of memory");
-                                goto out;
-                        }
-                        tmp_frame->local = fd;
-
-                        tmp_frame->root->pid = 0;
-                        tmp_frame->root->trans = conn;
-                        tmp_frame->root->lk_owner = 0;
-                        STACK_WIND (tmp_frame,
-                                    server_connection_cleanup_flush_cbk,
-                                    bound_xl, bound_xl->fops->flush, fd);
-                }
-        }
-
-        GF_FREE (fdentries);
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static int
-do_connection_cleanup (xlator_t *this, server_connection_t *conn,
-                       struct _lock_table *ltable, fdentry_t *fdentries, int fd_count)
-{
-        int             ret = 0;
-        int             saved_ret = 0;
-        call_frame_t   *frame = NULL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (frame == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-
-        saved_ret = do_lock_table_cleanup (this, conn, frame, ltable);
-
-        if (fdentries != NULL) {
-                ret = do_fd_cleanup (this, conn, frame, fdentries, fd_count);
-        }
-
-        STACK_DESTROY (frame->root);
-
-        if (saved_ret || ret) {
-                ret = -1;
-        }
-
-out:
-        return ret;
-}
-
-
-int
-gf_server_connection_cleanup (xlator_t *this, server_connection_t *conn)
-{
-        char                do_cleanup = 0;
-        struct _lock_table *ltable = NULL;
-        fdentry_t          *fdentries = NULL;
-        uint32_t            fd_count = 0;
-        int                 ret = 0; 
-
-        if (conn == NULL) {
-                goto out;
-        }
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                conn->active_transports--;
-                if (conn->active_transports == 0) {
-                        if (conn->ltable) {
-                                ltable = conn->ltable;
-                                conn->ltable = gf_lock_table_new ();
-                        }
-
-                        if (conn->fdtable) {
-                                fdentries = gf_fd_fdtable_get_all_fds (conn->fdtable,
-                                                                       &fd_count);
-                        }
-                        do_cleanup = 1;
-                }
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (do_cleanup && conn->bound_xl)
-                ret = do_connection_cleanup (this, conn, ltable, fdentries, fd_count);
-
-out:
-        return ret;
-}
-
-
-static int
-server_connection_destroy (xlator_t *this, server_connection_t *conn)
-{
-        call_frame_t       *frame = NULL, *tmp_frame = NULL;
-        xlator_t           *bound_xl = NULL;
-        int32_t             ret = -1;
-        struct list_head    file_lockers;
-        struct list_head    dir_lockers;
-        struct _lock_table *ltable = NULL;
-        struct _locker     *locker = NULL, *tmp = NULL;
-        struct gf_flock        flock = {0,};
-        fd_t               *fd = NULL;
-        int32_t             i = 0;
-        fdentry_t          *fdentries = NULL;
-        uint32_t             fd_count = 0;
-
-        if (conn == NULL) {
-                ret = 0;
-                goto out;
-        }
-
-        bound_xl = (xlator_t *) (conn->bound_xl);
-
-        if (bound_xl) {
-                /* trans will have ref_count = 1 after this call, but its
-                   ok since this function is called in
-                   GF_EVENT_TRANSPORT_CLEANUP */
-                frame = create_frame (this, this->ctx->pool);
-
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        if (conn->ltable) {
-                                ltable = conn->ltable;
-                                conn->ltable = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conn->lock);
-
-                INIT_LIST_HEAD (&file_lockers);
-                INIT_LIST_HEAD (&dir_lockers);
-
-                if (ltable) {
-                        LOCK (&ltable->lock);
-                        {
-                                list_splice_init (&ltable->file_lockers,
-                                                  &file_lockers);
-
-                                list_splice_init (&ltable->dir_lockers, &dir_lockers);
-                        }
-                        UNLOCK (&ltable->lock);
-                        GF_FREE (ltable);
-                }
-
-                flock.l_type  = F_UNLCK;
-                flock.l_start = 0;
-                flock.l_len   = 0;
-                list_for_each_entry_safe (locker,
-                                          tmp, &file_lockers, lockers) {
-                        tmp_frame = copy_frame (frame);
-                        /*
-                           pid = 0 is a special case that tells posix-locks
-                           to release all locks from this transport
-                        */
-                        tmp_frame->root->pid = 0;
-                        tmp_frame->root->trans = conn;
-
-                        if (locker->fd) {
-                                STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                            bound_xl,
-                                            bound_xl->fops->finodelk,
-                                            locker->volume,
-                                            locker->fd, F_SETLK, &flock);
-                                fd_unref (locker->fd);
-                        } else {
-                                STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                            bound_xl,
-                                            bound_xl->fops->inodelk,
-                                            locker->volume,
-                                            &(locker->loc), F_SETLK, &flock);
-                                loc_wipe (&locker->loc);
-                        }
-
-                        GF_FREE (locker->volume);
-
-			list_del_init (&locker->lockers);
-			GF_FREE (locker);
-		}
-
-                tmp = NULL;
-                locker = NULL;
-                list_for_each_entry_safe (locker, tmp, &dir_lockers, lockers) {
-                        tmp_frame = copy_frame (frame);
-
-                        tmp_frame->root->pid = 0;
-                        tmp_frame->root->trans = conn;
-
-                        if (locker->fd) {
-                                STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                            bound_xl,
-                                            bound_xl->fops->fentrylk,
-                                            locker->volume,
-                                            locker->fd, NULL,
-                                            ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                                fd_unref (locker->fd);
-                        } else {
-                                STACK_WIND (tmp_frame, gf_server_nop_cbk,
-                                            bound_xl,
-                                            bound_xl->fops->entrylk,
-                                            locker->volume,
-                                            &(locker->loc), NULL,
-                                            ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                                loc_wipe (&locker->loc);
-                        }
-
-                        GF_FREE (locker->volume);
-
-
-			list_del_init (&locker->lockers);
-			GF_FREE (locker);
-		}
-
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        if (conn->fdtable) {
-                                fdentries = gf_fd_fdtable_get_all_fds (conn->fdtable,
-                                                                       &fd_count);
-                                gf_fd_fdtable_destroy (conn->fdtable);
-                                conn->fdtable = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conn->lock);
-
-                if (fdentries != NULL) {
-                        for (i = 0; i < fd_count; i++) {
-                                fd = fdentries[i].fd;
-                                if (fd != NULL) {
-                                        tmp_frame = copy_frame (frame);
-                                        tmp_frame->local = fd;
-
-                                        STACK_WIND (tmp_frame,
-                                                    server_connection_cleanup_flush_cbk,
-                                                    bound_xl,
-                                                    bound_xl->fops->flush,
-                                                    fd);
-                                }
-                        }
-                        GF_FREE (fdentries);
-                }
-        }
-
-        if (frame) {
-                STACK_DESTROY (frame->root);
-        }
-
-        gf_log (this->name, GF_LOG_INFO, "destroyed connection of %s",
-                conn->id);
-
-	GF_FREE (conn->id);
-	GF_FREE (conn);
-
-out:
-        return ret;
-}
-
-
-server_connection_t *
-gf_server_connection_get (xlator_t *this, const char *id)
-{
-	server_connection_t *conn = NULL;
-	server_connection_t *trav = NULL;
-	server_conf_t       *conf = NULL;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                list_for_each_entry (trav, &conf->conns, list) {
-                        if (!strcmp (id, trav->id)) {
-                                conn = trav;
-                                break;
-                        }
-                }
-
-		if (!conn) {
-			conn = (void *) GF_CALLOC (1, sizeof (*conn),
-                                        gf_server_mt_server_connection_t);
-
-			conn->id = gf_strdup (id);
-			conn->fdtable = gf_fd_fdtable_alloc ();
-			conn->ltable  = gf_lock_table_new ();
-
-                        pthread_mutex_init (&conn->lock, NULL);
-
-			list_add (&conn->list, &conf->conns);
-		}
-
-                conn->ref++;
-                conn->active_transports++;
-	}
-	pthread_mutex_unlock (&conf->mutex);
-
-        return conn;
-}
-
-
-void
-gf_server_connection_put (xlator_t *this, server_connection_t *conn)
-{
-        server_conf_t       *conf = NULL;
-        server_connection_t *todel = NULL;
-
-        if (conn == NULL) {
-                goto out;
-        }
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->mutex);
-        {
-                conn->ref--;
-
-                if (!conn->ref) {
-                        list_del_init (&conn->list);
-                        todel = conn;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        if (todel) {
-                server_connection_destroy (this, todel);
-        }
-
-out:
-        return;
-}
diff --git a/xlators/protocol/legacy/server/src/server-helpers.h b/xlators/protocol/legacy/server/src/server-helpers.h
deleted file mode 100644
index cda0c36ef..000000000
--- a/xlators/protocol/legacy/server/src/server-helpers.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __SERVER_HELPERS_H__
-#define __SERVER_HELPERS_H__
-
-#define CALL_STATE(frame)   ((server_state_t *)frame->root->state)
-
-#define BOUND_XL(frame)     ((xlator_t *) CALL_STATE(frame)->bound_xl)
-
-#define TRANSPORT_FROM_FRAME(frame) ((transport_t *) CALL_STATE(frame)->trans)
-
-#define SERVER_CONNECTION(frame)  \
-	((server_connection_t *) TRANSPORT_FROM_FRAME(frame)->xl_private)
-
-#define SERVER_CONF(frame) \
-	((server_conf_t *)TRANSPORT_FROM_FRAME(frame)->xl->private)
-
-#define TRANSPORT_FROM_XLATOR(this) ((((server_conf_t *)this->private))->trans)
-
-#define INODE_LRU_LIMIT(this)						\
-	(((server_conf_t *)(this->private))->inode_lru_limit)
-
-#define IS_ROOT_INODE(inode) (inode == inode->table->root)
-
-#define IS_NOT_ROOT(pathlen) ((pathlen > 2)? 1 : 0)
-
-void free_old_server_state (server_state_t *state);
-
-void old_server_loc_wipe (loc_t *loc);
-
-#endif /* __SERVER_HELPERS_H__ */
diff --git a/xlators/protocol/legacy/server/src/server-mem-types.h b/xlators/protocol/legacy/server/src/server-mem-types.h
deleted file mode 100644
index 5e15fd90f..000000000
--- a/xlators/protocol/legacy/server/src/server-mem-types.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __AFR_MEM_TYPES_H__
-#define __AFR_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_server_mem_types_ {
-        gf_server_mt_dir_entry_t = gf_common_mt_end + 1,
-        gf_server_mt_volfile_ctx,
-        gf_server_mt_server_state_t,
-        gf_server_mt_server_conf_t,
-        gf_server_mt_locker,
-        gf_server_mt_lock_table,
-        gf_server_mt_char,
-        gf_server_mt_server_connection_t,
-        gf_server_mt_resolve_comp,
-        gf_server_mt_end
-};
-#endif
-
diff --git a/xlators/protocol/legacy/server/src/server-protocol.c b/xlators/protocol/legacy/server/src/server-protocol.c
deleted file mode 100644
index abd339542..000000000
--- a/xlators/protocol/legacy/server/src/server-protocol.c
+++ /dev/null
@@ -1,6587 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-#include <time.h>
-#include <sys/uio.h>
-#include <sys/resource.h>
-
-#include <libgen.h>
-
-#include "transport.h"
-#include "fnmatch.h"
-#include "xlator.h"
-#include "protocol.h"
-#include "server-protocol.h"
-#include "server-helpers.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "list.h"
-#include "dict.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "md5.h"
-
-
-static void
-print_caller (char *str, int size, call_frame_t *frame)
-{
-        int              filled = 0;
-        server_state_t  *state = NULL;
-        transport_t     *trans = NULL;
-
-        state = CALL_STATE (frame);
-        trans = state->trans;
-
-        filled += snprintf (str + filled, size - filled,
-                            " Callid=%"PRId64", Client=%s",
-                            frame->root->unique,
-                            trans->peerinfo.identifier);
-
-        return;
-}
-
-
-static void
-server_print_resolve (char *str, int size, server_resolve_t *resolve)
-{
-        int filled = 0;
-
-        if (!resolve) {
-                snprintf (str, size, "<nul>");
-                return;
-        }
-
-        filled += snprintf (str + filled, size - filled,
-                            " Resolve={");
-        if (resolve->fd_no != -1)
-                filled += snprintf (str + filled, size - filled,
-                                    "fd=%"PRId64",", (uint64_t) resolve->fd_no);
-        if (resolve->ino)
-                filled += snprintf (str + filled, size - filled,
-                                    "ino=%"PRIu64",", (uint64_t) resolve->ino);
-        if (resolve->par)
-                filled += snprintf (str + filled, size - filled,
-                                    "par=%"PRIu64",", (uint64_t) resolve->par);
-        if (resolve->gen)
-                filled += snprintf (str + filled, size - filled,
-                                    "gen=%"PRIu64",", (uint64_t) resolve->gen);
-        if (resolve->bname)
-                filled += snprintf (str + filled, size - filled,
-                                    "bname=%s,", resolve->bname);
-        if (resolve->path)
-                filled += snprintf (str + filled, size - filled,
-                                    "path=%s", resolve->path);
-
-        filled += snprintf (str + filled, size - filled, "}");
-}
-
-
-static void
-server_print_loc (char *str, int size, loc_t *loc)
-{
-        int filled = 0;
-
-        if (!loc) {
-                snprintf (str, size, "<nul>");
-                return;
-        }
-
-        filled += snprintf (str + filled, size - filled,
-                            " Loc={");
-
-        if (loc->path)
-                filled += snprintf (str + filled, size - filled,
-                                    "path=%s,", loc->path);
-        if (loc->inode)
-                filled += snprintf (str + filled, size - filled,
-                                    "inode=%p,", loc->inode);
-        if (loc->parent)
-                filled += snprintf (str + filled, size - filled,
-                                    "parent=%p", loc->parent);
-
-        filled += snprintf (str + filled, size - filled, "}");
-}
-
-
-static void
-server_print_params (char *str, int size, server_state_t *state)
-{
-        int filled = 0;
-
-        filled += snprintf (str + filled, size - filled,
-                            " Params={");
-
-        if (state->fd)
-                filled += snprintf (str + filled, size - filled,
-                                    "fd=%p,", state->fd);
-        if (state->valid)
-                filled += snprintf (str + filled, size - filled,
-                                    "valid=%d,", state->valid);
-        if (state->flags)
-                filled += snprintf (str + filled, size - filled,
-                                    "flags=%d,", state->flags);
-        if (state->wbflags)
-                filled += snprintf (str + filled, size - filled,
-                                    "wbflags=%d,", state->wbflags);
-        if (state->size)
-                filled += snprintf (str + filled, size - filled,
-                                    "size=%zu,", state->size);
-        if (state->offset)
-                filled += snprintf (str + filled, size - filled,
-                                    "offset=%"PRId64",", state->offset);
-        if (state->cmd)
-                filled += snprintf (str + filled, size - filled,
-                                    "cmd=%d,", state->cmd);
-        if (state->type)
-                filled += snprintf (str + filled, size - filled,
-                                    "type=%d,", state->type);
-        if (state->name)
-                filled += snprintf (str + filled, size - filled,
-                                    "name=%s,", state->name);
-        if (state->mask)
-                filled += snprintf (str + filled, size - filled,
-                                    "mask=%d,", state->mask);
-        if (state->volume)
-                filled += snprintf (str + filled, size - filled,
-                                    "volume=%s,", state->volume);
-
-        filled += snprintf (str + filled, size - filled,
-                            "bound_xl=%s}", state->bound_xl->name);
-}
-
-
-static int
-server_resolve_is_empty (server_resolve_t *resolve)
-{
-        if (resolve->fd_no != -1)
-                return 0;
-
-        if (resolve->ino != 0)
-                return 0;
-
-        if (resolve->gen != 0)
-                return 0;
-
-        if (resolve->par != 0)
-                return 0;
-
-        if (resolve->path != 0)
-                return 0;
-
-        if (resolve->bname != 0)
-                return 0;
-
-        return 1;
-}
-
-void
-gf_server_print_request (call_frame_t *frame)
-{
-        server_conf_t   *conf = NULL;
-        xlator_t        *this = NULL;
-        server_state_t  *state = NULL;
-        char             resolve_vars[256];
-        char             resolve2_vars[256];
-        char             loc_vars[256];
-        char             loc2_vars[256];
-        char             other_vars[512];
-        char             caller[512];
-        char            *op = "UNKNOWN";
-
-        this = frame->this;
-        conf = this->private;
-
-        state = CALL_STATE (frame);
-
-        if (!conf->trace)
-                return;
-
-        memset (resolve_vars, '\0', 256);
-        memset (resolve2_vars, '\0', 256);
-        memset (loc_vars, '\0', 256);
-        memset (loc2_vars, '\0', 256);
-        memset (other_vars, '\0', 256);
-
-        print_caller (caller, 256, frame);
-
-        if (!server_resolve_is_empty (&state->resolve)) {
-                server_print_resolve (resolve_vars, 256, &state->resolve);
-                server_print_loc (loc_vars, 256, &state->loc);
-        }
-
-        if (!server_resolve_is_empty (&state->resolve2)) {
-                server_print_resolve (resolve2_vars, 256, &state->resolve2);
-                server_print_loc (loc2_vars, 256, &state->loc2);
-        }
-
-        server_print_params (other_vars, 512, state);
-
-        switch (frame->root->type) {
-        case GF_OP_TYPE_FOP_REQUEST:
-        case GF_OP_TYPE_FOP_REPLY:
-                op = gf_fop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_MOP_REQUEST:
-        case GF_OP_TYPE_MOP_REPLY:
-                op = gf_mop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_CBK_REQUEST:
-        case GF_OP_TYPE_CBK_REPLY:
-                op = gf_cbk_list[frame->root->op];
-                break;
-        }
-
-        gf_log (this->name, GF_LOG_INFO,
-                "%s%s%s%s%s%s%s",
-                gf_fop_list[frame->root->op], caller,
-                resolve_vars, loc_vars, resolve2_vars, loc2_vars, other_vars);
-}
-
-
-static void
-server_print_reply (call_frame_t *frame, int op_ret, int op_errno)
-{
-        server_conf_t   *conf = NULL;
-        server_state_t  *state = NULL;
-        xlator_t        *this = NULL;
-        char             caller[512];
-        char             fdstr[32];
-        char            *op = "UNKNOWN";
-
-        this = frame->this;
-        conf = this->private;
-
-        if (!conf->trace)
-                return;
-
-        state = CALL_STATE (frame);
-
-        print_caller (caller, 256, frame);
-
-        switch (frame->root->type) {
-        case GF_OP_TYPE_FOP_REQUEST:
-        case GF_OP_TYPE_FOP_REPLY:
-                op = gf_fop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_MOP_REQUEST:
-        case GF_OP_TYPE_MOP_REPLY:
-                op = gf_mop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_CBK_REQUEST:
-        case GF_OP_TYPE_CBK_REPLY:
-                op = gf_cbk_list[frame->root->op];
-                break;
-        }
-
-        fdstr[0] = '\0';
-        if (state->fd)
-                snprintf (fdstr, 32, " fd=%p", state->fd);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "%s%s => (%d, %d)%s",
-                op, caller, op_ret, op_errno, fdstr);
-}
-
-
-
-static void
-protocol_server_reply (call_frame_t *frame, int type, int op,
-                       gf_hdr_common_t *hdr, size_t hdrlen,
-                       struct iovec *vector, int count,
-                       struct iobref *iobref)
-{
-        server_state_t *state = NULL;
-        xlator_t       *bound_xl = NULL;
-        transport_t    *trans = NULL;
-        int             ret = 0;
-
-        xlator_t       *this = NULL;
-
-        bound_xl = BOUND_XL (frame);
-        state    = CALL_STATE (frame);
-        trans    = state->trans;
-        this     = frame->this;
-
-        hdr->callid = hton64 (frame->root->unique);
-        hdr->type   = hton32 (type);
-        hdr->op     = hton32 (op);
-
-        server_print_reply (frame, ntoh32 (hdr->rsp.op_ret),
-                            gf_error_to_errno (ntoh32 (hdr->rsp.op_errno)));
-
-        ret = transport_submit (trans, (char *)hdr, hdrlen, vector,
-                                count, iobref);
-        if (ret < 0) {
-                gf_log ("protocol/server", GF_LOG_ERROR,
-                        "frame %"PRId64": failed to submit. op= %d, type= %d",
-                        frame->root->unique, op, type);
-        }
-
-        STACK_DESTROY (frame->root);
-
-        if (state)
-                free_old_server_state (state);
-
-}
-
-
-static int
-gf_add_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc, fd_t *fd, pid_t pid)
-{
-        int32_t         ret = -1;
-        struct _locker *new = NULL;
-        uint8_t         dir = 0;
-
-	new = GF_CALLOC (1, sizeof (struct _locker),
-                         gf_server_mt_locker);
-        if (new == NULL) {
-                gf_log ("server", GF_LOG_ERROR,
-                        "failed to allocate memory for \'struct _locker\'");
-                goto out;
-        }
-        INIT_LIST_HEAD (&new->lockers);
-
-        new->volume = gf_strdup (volume);
-
-        if (fd == NULL) {
-                loc_copy (&new->loc, loc);
-                dir = IA_ISDIR (new->loc.inode->ia_type);
-        } else {
-                new->fd = fd_ref (fd);
-                dir = IA_ISDIR (fd->inode->ia_type);
-        }
-
-        new->pid = pid;
-
-        LOCK (&table->lock);
-        {
-                if (dir)
-                        list_add_tail (&new->lockers, &table->dir_lockers);
-                else
-                        list_add_tail (&new->lockers, &table->file_lockers);
-        }
-        UNLOCK (&table->lock);
-out:
-        return ret;
-}
-
-
-static int
-gf_del_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc, fd_t *fd, pid_t pid)
-{
-        struct _locker    *locker = NULL;
-        struct _locker    *tmp = NULL;
-        int32_t            ret = 0;
-        uint8_t            dir = 0;
-        struct list_head  *head = NULL;
-        struct list_head   del;
-
-        INIT_LIST_HEAD (&del);
-
-        if (fd) {
-                dir = IA_ISDIR (fd->inode->ia_type);
-        } else {
-                dir = IA_ISDIR (loc->inode->ia_type);
-        }
-
-        LOCK (&table->lock);
-        {
-                if (dir) {
-                        head = &table->dir_lockers;
-                } else {
-                        head = &table->file_lockers;
-                }
-
-                list_for_each_entry_safe (locker, tmp, head, lockers) {
-                        if (locker->fd && fd &&
-                            (locker->fd == fd) && (locker->pid == pid)
-                            && !strcmp (locker->volume, volume)) {
-                                list_move_tail (&locker->lockers, &del);
-                        } else if (locker->loc.inode &&
-                                   loc &&
-                                   (locker->loc.inode == loc->inode) &&
-                                   (locker->pid == pid)
-                                   && !strcmp (locker->volume, volume)) {
-                                list_move_tail (&locker->lockers, &del);
-                        }
-                }
-        }
-        UNLOCK (&table->lock);
-
-        tmp = NULL;
-        locker = NULL;
-
-        list_for_each_entry_safe (locker, tmp, &del, lockers) {
-                list_del_init (&locker->lockers);
-                if (locker->fd)
-                        fd_unref (locker->fd);
-                else
-                        loc_wipe (&locker->loc);
-
-                GF_FREE (locker->volume);
-		GF_FREE (locker);
-	}
-
-        return ret;
-}
-
-
-/*
- * server_lk_cbk - lk callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @lock:
- *
- * not for external reference
- */
-static int
-server_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_lk_rsp_t     *rsp = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-        server_state_t      *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_flock_from_flock (&rsp->flock, lock);
-        } else if (op_errno != ENOSYS) {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": LK %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t  *conn = NULL;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_inodelk_rsp_t *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-
-        conn = SERVER_CONNECTION(frame);
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                if (state->flock.l_type == F_UNLCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid);
-        } else if (op_errno != ENOSYS) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": INODELK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_INODELK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t   *conn = NULL;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_finodelk_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        conn = SERVER_CONNECTION(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                if (state->flock.l_type == F_UNLCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       NULL, state->fd,
-                                       frame->root->pid);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       NULL, state->fd,
-                                       frame->root->pid);
-        } else if (op_errno != ENOSYS) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": FINODELK %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FINODELK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_entrylk_cbk -
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @lock:
- *
- * not for external reference
- */
-static int
-server_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t  *conn = NULL;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_entrylk_rsp_t *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-
-        conn = SERVER_CONNECTION(frame);
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                if (state->cmd == ENTRYLK_UNLOCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid);
-        } else if (op_errno != ENOSYS) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": INODELK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_ENTRYLK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t   *conn = NULL;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fentrylk_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        conn = SERVER_CONNECTION(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                if (state->cmd == ENTRYLK_UNLOCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       NULL, state->fd, frame->root->pid);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       NULL, state->fd, frame->root->pid);
-        } else if (op_errno != ENOSYS) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": FENTRYLK %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FENTRYLK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_access_cbk - access callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_access_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_ACCESS,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_rmdir_cbk - rmdir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_rmdir_rsp_t *rsp = NULL;
-        server_state_t     *state = NULL;
-        int32_t             gf_errno = 0;
-        size_t              hdrlen = 0;
-        inode_t            *parent = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-                parent = inode_parent (state->loc.inode, 0, NULL);
-                if (parent)
-                        inode_unref (parent);
-                else
-                        inode_forget (state->loc.inode, 0);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": RMDIR %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_RMDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_mkdir_cbk - mkdir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_mkdir_rsp_t *rsp = NULL;
-        server_state_t     *state = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        inode_t            *link_inode = NULL;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": MKDIR %s  ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_MKDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_mknod_cbk - mknod callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_mknod_rsp_t *rsp = NULL;
-        server_state_t     *state = NULL;
-        int32_t             gf_errno = 0;
-        size_t              hdrlen = 0;
-        inode_t            *link_inode = NULL;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": MKNOD %s ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_MKNOD,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_fsyncdir_cbk - fsyncdir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fsyncdir_rsp_t *rsp = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-        server_state_t        *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        if (op_ret < 0) {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": FSYNCDIR %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSYNCDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-
-/*
- * server_readdir_cbk - getdents callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_readdir_rsp_t *rsp = NULL;
-        size_t                hdrlen = 0;
-        size_t                buf_size = 0;
-        int32_t               gf_errno = 0;
-        server_state_t       *state = NULL;
-
-        if (op_ret > 0)
-                buf_size = gf_dirent_serialize (entries, NULL, 0);
-
-        hdrlen = gf_hdr_len (rsp, buf_size);
-        hdr    = gf_hdr_new (rsp, buf_size);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret > 0) {
-                rsp->size = hton32 (buf_size);
-                gf_dirent_serialize (entries, rsp->buf, buf_size);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": READDIR %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_READDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_releasedir_cbk - releasedir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_releasedir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t         *hdr = NULL;
-        gf_cbk_releasedir_rsp_t *rsp = NULL;
-        size_t                   hdrlen = 0;
-        int32_t                  gf_errno = 0;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_CBK_REPLY, GF_CBK_RELEASEDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_opendir_cbk - opendir callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- * @fd: file descriptor structure of opened directory
- *
- * not for external reference
- */
-static int
-server_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        server_connection_t   *conn = NULL;
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_opendir_rsp_t  *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-        uint64_t               fd_no = 0;
-
-        conn = SERVER_CONNECTION (frame);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd); // on behalf of the client
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": OPENDIR %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        rsp->fd           = hton64 (fd_no);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_OPENDIR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_statfs_cbk - statfs callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- * @buf:
- *
- * not for external reference
- */
-static int
-server_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *buf)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_statfs_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_statfs_from_statfs (&rsp->statfs, buf);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_STATFS,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_removexattr_cbk - removexattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t          *hdr = NULL;
-        gf_fop_removexattr_rsp_t *rsp = NULL;
-        server_state_t           *state = NULL;
-        size_t                    hdrlen = 0;
-        int32_t                   gf_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_REMOVEXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_getxattr_cbk - getxattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- * @value:
- *
- * not for external reference
- */
-static int
-server_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_getxattr_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                len = 0;
-        int32_t                gf_errno = 0;
-        int32_t                ret = -1;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                len = dict_serialized_length (dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to get serialized length of "
-                                "reply dict",
-                                state->loc.path, state->resolve.ino);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, len + 1);
-        hdr    = gf_hdr_new (rsp, len + 1);
-        rsp    = gf_param (hdr);
-
-        if (op_ret >= 0) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to serialize reply dict",
-                                state->loc.path, state->resolve.ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                }
-        }
-        rsp->dict_len = hton32 (len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_GETXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gf_hdr_common_t        *hdr  = NULL;
-        gf_fop_fgetxattr_rsp_t *rsp = NULL;
-        server_state_t         *state = NULL;
-        size_t                  hdrlen = 0;
-        int32_t                 len = 0;
-        int32_t                 gf_errno = 0;
-        int32_t                 ret = -1;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                len = dict_serialized_length (dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to get serialized length of "
-                                "reply dict",
-                                state->loc.path, state->resolve.ino);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, len + 1);
-        hdr    = gf_hdr_new (rsp, len + 1);
-        rsp    = gf_param (hdr);
-
-        if (op_ret >= 0) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to serialize reply dict",
-                                state->loc.path, state->resolve.ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                }
-        }
-        rsp->dict_len = hton32 (len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FGETXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_setxattr_cbk - setxattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_setxattr_rsp_t  *rsp = NULL;
-        server_state_t         *state = NULL;
-        size_t                  hdrlen = 0;
-        int32_t                 gf_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_SETXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_fsetxattr_rsp_t *rsp = NULL;
-        server_state_t         *state = NULL;
-        size_t                  hdrlen = 0;
-        int32_t                 gf_errno = 0;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSETXATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_rename_cbk - rename callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_rename_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                stbuf->ia_ino  = state->loc.inode->ino;
-                stbuf->ia_type = state->loc.inode->ia_type;
-
-                gf_log (state->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": RENAME_CBK (%"PRId64") %"PRId64"/%s "
-                        "==> %"PRId64"/%s",
-                        frame->root->unique, state->loc.inode->ino,
-                        state->loc.parent->ino, state->loc.name,
-                        state->loc2.parent->ino, state->loc2.name);
-
-                inode_rename (state->itable,
-                              state->loc.parent, state->loc.name,
-                              state->loc2.parent, state->loc2.name,
-                              state->loc.inode, stbuf);
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-
-                gf_stat_from_iatt (&rsp->preoldparent, preoldparent);
-                gf_stat_from_iatt (&rsp->postoldparent, postoldparent);
-
-                gf_stat_from_iatt (&rsp->prenewparent, prenewparent);
-                gf_stat_from_iatt (&rsp->postnewparent, postnewparent);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_RENAME,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_unlink_cbk - unlink callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_unlink_rsp_t  *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-        inode_t              *parent = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                gf_log (state->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": UNLINK_CBK %"PRId64"/%s (%"PRId64")",
-                        frame->root->unique, state->loc.parent->ino,
-                        state->loc.name, state->loc.inode->ino);
-
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-
-                parent = inode_parent (state->loc.inode, 0, NULL);
-                if (parent)
-                        inode_unref (parent);
-                else
-                        inode_forget (state->loc.inode, 0);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": UNLINK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_UNLINK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_symlink_cbk - symlink callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret: return value
- * @op_errno: errno
- *
- * not for external reference
- */
-static int
-server_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_symlink_rsp_t *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-        inode_t              *link_inode = NULL;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": SYMLINK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_SYMLINK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_link_cbk - link callback for server protocol
- * @frame: call frame
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent)
-{
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_link_rsp_t *rsp = NULL;
-        server_state_t    *state = NULL;
-        int32_t            gf_errno = 0;
-        size_t             hdrlen = 0;
-        inode_t           *link_inode = NULL;
-
-        state = CALL_STATE(frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                stbuf->ia_ino = state->loc.inode->ino;
-
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-                gf_log (state->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": LINK (%"PRId64") %"PRId64"/%s ==> %"PRId64"/%s",
-                        frame->root->unique, inode->ino,
-                        state->loc2.parent->ino,
-                        state->loc2.name, state->loc.parent->ino,
-                        state->loc.name);
-
-                link_inode = inode_link (inode, state->loc2.parent,
-                                         state->loc2.name, stbuf);
-                inode_unref (link_inode);
-        } else {
-                gf_log (state->bound_xl->name, GF_LOG_DEBUG,
-                        "%"PRId64": LINK (%"PRId64") %"PRId64"/%s ==> %"PRId64"/%s "
-                        " ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve2.ino,
-                        state->resolve2.par,
-                        state->resolve2.bname, state->resolve.par,
-                        state->resolve.bname,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LINK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_truncate_cbk - truncate callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_truncate_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->prestat, prebuf);
-                gf_stat_from_iatt (&rsp->poststat, postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": TRUNCATE %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_TRUNCATE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_fstat_cbk - fstat callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_fstat_rsp_t *rsp = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        server_state_t     *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FSTAT %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSTAT,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_ftruncate_cbk - ftruncate callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-        gf_hdr_common_t        *hdr = NULL;
-        gf_fop_ftruncate_rsp_t *rsp = NULL;
-        size_t                  hdrlen = 0;
-        int32_t                 gf_errno = 0;
-        server_state_t         *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->prestat, prebuf);
-                gf_stat_from_iatt (&rsp->poststat, postbuf);
-        } else {
-                state = CALL_STATE (frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FTRUNCATE %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FTRUNCATE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_flush_cbk - flush callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_flush_rsp_t *rsp = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        server_state_t     *state = NULL;
-
-        if (op_ret < 0) {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FLUSH %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FLUSH,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_fsync_cbk - fsync callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_fsync_rsp_t *rsp = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        server_state_t     *state = NULL;
-
-        if (op_ret < 0) {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FSYNC %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&(rsp->prestat), prebuf);
-                gf_stat_from_iatt (&(rsp->poststat), postbuf);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSYNC,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_release_cbk - rleease callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_release_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_cbk_release_rsp_t *rsp = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_CBK_REPLY, GF_CBK_RELEASE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_writev_cbk - writev callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-
-static int
-server_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-        gf_hdr_common_t    *hdr = NULL;
-        gf_fop_write_rsp_t *rsp = NULL;
-        size_t              hdrlen = 0;
-        int32_t             gf_errno = 0;
-        server_state_t     *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->prestat, prebuf);
-                gf_stat_from_iatt (&rsp->poststat, postbuf);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": WRITEV %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_WRITE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_readv_cbk - readv callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @vector:
- * @count:
- *
- * not for external reference
- */
-static int
-server_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count,
-                  struct iatt *stbuf, struct iobref *iobref)
-{
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_read_rsp_t *rsp = NULL;
-        size_t             hdrlen = 0;
-        int32_t            gf_errno = 0;
-        server_state_t    *state = NULL;
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": READV %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_READ,
-                               hdr, hdrlen, vector, count, iobref);
-
-        return 0;
-}
-
-
-/*
- * server_open_cbk - open callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @fd:
- *
- * not for external reference
- */
-static int
-server_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        server_connection_t  *conn = NULL;
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_open_rsp_t    *rsp = NULL;
-        server_state_t       *state = NULL;
-        size_t                hdrlen = 0;
-        int32_t               gf_errno = 0;
-        uint64_t              fd_no = 0;
-
-        conn = SERVER_CONNECTION (frame);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": OPEN %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        rsp->fd           = hton64 (fd_no);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_OPEN,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_create_cbk - create callback for server
- * @frame: call frame
- * @cookie:
- * @this:  translator structure
- * @op_ret:
- * @op_errno:
- * @fd: file descriptor
- * @inode: inode structure
- * @stbuf: struct iatt of created file
- *
- * not for external reference
- */
-static int
-server_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                   struct iatt *preparent, struct iatt *postparent)
-{
-        server_connection_t *conn = NULL;
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_create_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-        uint64_t             fd_no = 0;
-        inode_t             *link_inode = NULL;
-
-        conn = SERVER_CONNECTION (frame);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                gf_log (state->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": CREATE %"PRId64"/%s (%"PRId64")",
-                        frame->root->unique, state->loc.parent->ino,
-                        state->loc.name, stbuf->ia_ino);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-
-                if (link_inode != inode) {
-
-                        /*
-                           VERY racy code (if used anywhere else)
-                           -- don't do this without understanding
-                        */
-
-                        inode_unref (fd->inode);
-                        fd->inode = inode_ref (link_inode);
-                }
-
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd);
-
-                if ((fd_no < 0) || (fd == 0)) {
-                        op_ret = fd_no;
-                        op_errno = errno;
-                }
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": CREATE %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        rsp->fd           = hton64 (fd_no);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-                gf_stat_from_iatt (&rsp->preparent, preparent);
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_CREATE,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_readlink_cbk - readlink callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @buf:
- *
- * not for external reference
- */
-static int
-server_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, const char *buf,
-                     struct iatt *sbuf)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_readlink_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        size_t                 linklen = 0;
-        int32_t                gf_errno = 0;
-
-        state  = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                linklen = strlen (buf) + 1;
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": READLINK %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        hdrlen = gf_hdr_len (rsp, linklen);
-        hdr    = gf_hdr_new (rsp, linklen);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&(rsp->buf), sbuf);
-                strcpy (rsp->path, buf);
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_READLINK,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_stat_cbk - stat callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
-{
-        gf_hdr_common_t   *hdr = NULL;
-        gf_fop_stat_rsp_t *rsp = NULL;
-        server_state_t    *state = NULL;
-        size_t             hdrlen = 0;
-        int32_t            gf_errno = 0;
-
-        state  = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": STAT %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_STAT,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_setattr_cbk - setattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-
-static int
-server_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_setattr_rsp_t  *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        state  = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->statpre, statpre);
-                gf_stat_from_iatt (&rsp->statpost, statpost);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": SETATTR %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_SETATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/*
- * server_setattr_cbk - setattr callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_fsetattr_rsp_t *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        state  = CALL_STATE (frame);
-
-        hdrlen = gf_hdr_len (rsp, 0);
-        hdr    = gf_hdr_new (rsp, 0);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno_to_error (op_errno));
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp->statpre, statpre);
-                gf_stat_from_iatt (&rsp->statpost, statpost);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FSETATTR %"PRId64" (%"PRId64") ==> "
-                        "%"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FSETATTR,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * server_lookup_cbk - lookup callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @inode:
- * @stbuf:
- *
- * not for external reference
- */
-static int
-server_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *stbuf, dict_t *dict,
-                   struct iatt *postparent)
-{
-        gf_hdr_common_t     *hdr = NULL;
-        gf_fop_lookup_rsp_t *rsp = NULL;
-        server_state_t      *state = NULL;
-        inode_t             *root_inode = NULL;
-        int32_t              dict_len = 0;
-        size_t               hdrlen = 0;
-        int32_t              gf_errno = 0;
-        int32_t              ret = -1;
-        inode_t             *link_inode = NULL;
-        loc_t                fresh_loc = {0,};
-
-        state = CALL_STATE(frame);
-
-        if (state->is_revalidate == 1 && op_ret == -1) {
-                state->is_revalidate = 2;
-                loc_copy (&fresh_loc, &state->loc);
-                inode_unref (fresh_loc.inode);
-                fresh_loc.inode = inode_new (state->itable);
-
-                STACK_WIND (frame, server_lookup_cbk,
-                            BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
-                            &fresh_loc, state->dict);
-
-                loc_wipe (&fresh_loc);
-                return 0;
-        }
-
-        if (dict) {
-                dict_len = dict_serialized_length (dict);
-                if (dict_len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to get serialized "
-                                "length of reply dict",
-                                state->loc.path, state->loc.inode->ino);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        dict_len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, dict_len);
-        hdr    = gf_hdr_new (rsp, dict_len);
-        rsp    = gf_param (hdr);
-
-        if ((op_ret >= 0) && dict) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to serialize reply dict",
-                                state->loc.path, state->loc.inode->ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                        dict_len = 0;
-                }
-        }
-        rsp->dict_len = hton32 (dict_len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (postparent)
-                gf_stat_from_iatt (&rsp->postparent, postparent);
-
-        if (op_ret == 0) {
-                root_inode = BOUND_XL(frame)->itable->root;
-                if (inode == root_inode) {
-                        /* we just looked up root ("/") */
-                        stbuf->ia_ino = 1;
-                        if (inode->ia_type == 0)
-                                inode->ia_type = stbuf->ia_type;
-                }
-
-                gf_stat_from_iatt (&rsp->stat, stbuf);
-
-                if (inode->ino != 1) {
-                        link_inode = inode_link (inode, state->loc.parent,
-                                                 state->loc.name, stbuf);
-                        inode_lookup (link_inode);
-                        inode_unref (link_inode);
-                }
-        } else {
-                if (state->is_revalidate && op_errno == ENOENT) {
-                        if (state->loc.inode->ino != 1) {
-                                inode_unlink (state->loc.inode,
-                                              state->loc.parent,
-                                              state->loc.name);
-                        }
-                }
-
-                gf_log (this->name,
-                        (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_DEBUG),
-                        "%"PRId64": LOOKUP %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LOOKUP,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_xattrop_rsp_t  *rsp = NULL;
-        server_state_t        *state = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                len = 0;
-        int32_t                gf_errno = 0;
-        int32_t                ret = -1;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": XATTROP %s (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? state->loc.inode->ino : 0,
-                        op_ret, strerror (op_errno));
-        }
-
-        if ((op_ret >= 0) && dict) {
-                len = dict_serialized_length (dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to get serialized length"
-                                " for reply dict",
-                                state->loc.path, state->loc.inode->ino);
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, len + 1);
-        hdr    = gf_hdr_new (rsp, len + 1);
-        rsp    = gf_param (hdr);
-
-        if ((op_ret >= 0) && dict) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s (%"PRId64"): failed to serialize reply dict",
-                                state->loc.path, state->loc.inode->ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                        len = 0;
-                }
-        }
-        rsp->dict_len = hton32 (len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_XATTROP,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gf_hdr_common_t      *hdr = NULL;
-        gf_fop_xattrop_rsp_t *rsp = NULL;
-        size_t                hdrlen = 0;
-        int32_t               len = 0;
-        int32_t               gf_errno = 0;
-        int32_t               ret = -1;
-        server_state_t       *state = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FXATTROP %"PRId64" (%"PRId64") ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        if ((op_ret >= 0) && dict) {
-                len = dict_serialized_length (dict);
-                if (len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "fd - %"PRId64" (%"PRId64"): failed to get "
-                                "serialized length for reply dict",
-                                state->resolve.fd_no, state->fd->inode->ino);
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        len = 0;
-                }
-        }
-
-        hdrlen = gf_hdr_len (rsp, len + 1);
-        hdr    = gf_hdr_new (rsp, len + 1);
-        rsp    = gf_param (hdr);
-
-        if ((op_ret >= 0) && dict) {
-                ret = dict_serialize (dict, rsp->dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "fd - %"PRId64" (%"PRId64"): failed to "
-                                "serialize reply dict",
-                                state->resolve.fd_no, state->fd->inode->ino);
-                        op_ret = -1;
-                        op_errno = -ret;
-                        len = 0;
-                }
-        }
-        rsp->dict_len = hton32 (len);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_FXATTROP,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_lookup_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        if (!state->loc.inode)
-                state->loc.inode = inode_new (state->itable);
-        else
-                state->is_revalidate = 1;
-
-        STACK_WIND (frame, server_lookup_cbk,
-                    bound_xl, bound_xl->fops->lookup,
-                    &state->loc, state->dict);
-
-        return 0;
-err:
-        server_lookup_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_lookup (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_lookup_req_t *req = NULL;
-        server_state_t      *state = NULL;
-        int32_t              ret = -1;
-        size_t               pathlen = 0;
-        size_t               baselen = 0;
-        size_t               dictlen = 0;
-        dict_t              *xattr_req = NULL;
-        char                *req_dictbuf = NULL;
-
-        req = gf_param (hdr);
-
-        state = CALL_STATE (frame);
-
-        pathlen = STRLEN_0 (req->path);
-        dictlen = ntoh32 (req->dictlen);
-
-        /* NOTE: lookup() uses req->ino only to identify if a lookup()
-         *       is requested for 'root' or not
-         */
-        state->resolve.ino    = ntoh64 (req->ino);
-        if (state->resolve.ino != 1)
-                state->resolve.ino = 0;
-
-        state->resolve.type   = RESOLVE_DONTCARE;
-        state->resolve.par    = ntoh64 (req->par);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-
-        if (IS_NOT_ROOT (pathlen)) {
-                state->resolve.bname = gf_strdup (req->bname + pathlen);
-                baselen = STRLEN_0 (state->resolve.bname);
-        }
-
-        if (dictlen) {
-                /* Unserialize the dictionary */
-                req_dictbuf = memdup (req->dict + pathlen + baselen, dictlen);
-
-                xattr_req = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dictlen, &xattr_req);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "%"PRId64": %s (%"PRId64"): failed to "
-                                "unserialize req-buffer to dictionary",
-                                frame->root->unique, state->resolve.path,
-                                state->resolve.ino);
-                        GF_FREE (req_dictbuf);
-                        goto err;
-                }
-
-                xattr_req->extra_free = req_dictbuf;
-                state->dict = xattr_req;
-        }
-
-        gf_resolve_and_resume (frame, server_lookup_resume);
-
-        return 0;
-err:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        server_lookup_cbk (frame, NULL, frame->this, -1, EINVAL, NULL, NULL,
-                           NULL, NULL);
-        return 0;
-}
-
-
-/*
- * server_forget - forget function for server protocol
- *
- * not for external reference
- */
-static int
-server_forget (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_log ("forget", GF_LOG_CRITICAL, "function not implemented");
-        return 0;
-}
-
-
-static int
-server_stat_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_stat_cbk,
-                    bound_xl, bound_xl->fops->stat, &state->loc);
-        return 0;
-err:
-        server_stat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                         state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_stat (call_frame_t *frame, xlator_t *bound_xl,
-             gf_hdr_common_t *hdr, size_t hdrlen,
-             struct iobuf *iobuf)
-{
-        gf_fop_stat_req_t *req = NULL;
-        server_state_t    *state = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-        {
-                state->resolve.type  = RESOLVE_MUST;
-                state->resolve.ino   = ntoh64 (req->ino);
-                state->resolve.gen   = ntoh64 (req->gen);
-                state->resolve.path  = gf_strdup (req->path);
-        }
-
-        gf_resolve_and_resume (frame, server_stat_resume);
-
-        return 0;
-}
-
-
-static int
-server_setattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setattr_cbk,
-                    bound_xl, bound_xl->fops->setattr,
-                    &state->loc, &state->stbuf, state->valid);
-        return 0;
-err:
-        server_setattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_setattr (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_setattr_req_t *req = NULL;
-        server_state_t       *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.ino   = ntoh64 (req->ino);
-        state->resolve.gen   = ntoh64 (req->gen);
-        state->resolve.path  = gf_strdup (req->path);
-
-        gf_stat_to_iatt (&req->stbuf, &state->stbuf);
-        state->valid = ntoh32 (req->valid);
-
-        gf_resolve_and_resume (frame, server_setattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_fsetattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsetattr_cbk,
-                    bound_xl, bound_xl->fops->fsetattr,
-                    state->fd, &state->stbuf, state->valid);
-        return 0;
-err:
-        server_fsetattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fsetattr (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_fsetattr_req_t  *req = NULL;
-        server_state_t         *state = NULL;
-
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = ntoh64 (req->fd);
-
-        gf_stat_to_iatt (&req->stbuf, &state->stbuf);
-        state->valid = ntoh32 (req->valid);
-
-        gf_resolve_and_resume (frame, server_fsetattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_readlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readlink_cbk,
-                    bound_xl, bound_xl->fops->readlink,
-                    &state->loc, state->size);
-        return 0;
-err:
-        server_readlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_readlink (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_readlink_req_t *req = NULL;
-        server_state_t        *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.ino  = ntoh64 (req->ino);
-        state->resolve.gen  = ntoh64 (req->gen);
-        state->resolve.path = gf_strdup (req->path);
-
-        state->size  = ntoh32 (req->size);
-
-        gf_resolve_and_resume (frame, server_readlink_resume);
-
-        return 0;
-}
-
-
-static int
-server_create_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-        state->fd->flags = state->flags;
-
-        STACK_WIND (frame, server_create_cbk,
-                    bound_xl, bound_xl->fops->create,
-                    &(state->loc), state->flags, state->mode,
-                    state->fd, state->params);
-
-        return 0;
-err:
-        server_create_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL, NULL,
-                           NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_create (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_create_req_t *req = NULL;
-        server_state_t      *state = NULL;
-        int                  pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type   = RESOLVE_NOT;
-        state->resolve.par    = ntoh64 (req->par);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.bname  = gf_strdup (req->bname + pathlen);
-        state->mode           = ntoh32 (req->mode);
-        state->flags          = gf_flags_to_flags (ntoh32 (req->flags));
-
-        gf_resolve_and_resume (frame, server_create_resume);
-
-        return 0;
-}
-
-
-static int
-server_open_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-        state->fd->flags = state->flags;
-
-        STACK_WIND (frame, server_open_cbk,
-                    bound_xl, bound_xl->fops->open,
-                    &state->loc, state->flags, state->fd, 0);
-
-        return 0;
-err:
-        server_open_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                         state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_open (call_frame_t *frame, xlator_t *bound_xl,
-             gf_hdr_common_t *hdr, size_t hdrlen,
-             struct iobuf *iobuf)
-{
-        gf_fop_open_req_t  *req = NULL;
-        server_state_t     *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.ino   = ntoh64 (req->ino);
-        state->resolve.gen   = ntoh64 (req->gen);
-        state->resolve.path  = gf_strdup (req->path);
-
-        state->flags = gf_flags_to_flags (ntoh32 (req->flags));
-
-        gf_resolve_and_resume (frame, server_open_resume);
-
-        return 0;
-}
-
-
-static int
-server_readv_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readv_cbk,
-                    bound_xl, bound_xl->fops->readv,
-                    state->fd, state->size, state->offset);
-
-        return 0;
-err:
-        server_readv_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, 0, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_readv (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_read_req_t   *req = NULL;
-        server_state_t      *state = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = ntoh64 (req->fd);
-        state->size           = ntoh32 (req->size);
-        state->offset         = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_readv_resume);
-
-        return 0;
-}
-
-
-static int
-server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t   *state = NULL;
-        struct iovec      iov = {0, };
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        iov.iov_len  = state->size;
-
-        if (state->iobuf) {
-                iov.iov_base = state->iobuf->ptr;
-        }
-
-        STACK_WIND (frame, server_writev_cbk,
-                    bound_xl, bound_xl->fops->writev,
-                    state->fd, &iov, 1, state->offset, state->iobref);
-
-        return 0;
-err:
-        server_writev_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_writev (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_write_req_t  *req = NULL;
-        server_state_t      *state = NULL;
-        struct iobref       *iobref = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->offset        = ntoh64 (req->offset);
-        state->size          = ntoh32 (req->size);
-
-        if (iobuf) {
-                iobref = iobref_new ();
-                iobref_add (iobref, iobuf);
-
-                state->iobuf = iobuf;
-                state->iobref = iobref;
-        }
-
-        gf_resolve_and_resume (frame, server_writev_resume);
-
-        return 0;
-}
-
-
-static int
-server_release (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_cbk_release_req_t  *req = NULL;
-        server_state_t        *state = NULL;
-        server_connection_t   *conn = NULL;
-
-        conn  = SERVER_CONNECTION (frame);
-        state = CALL_STATE (frame);
-        req   = gf_param (hdr);
-
-        state->resolve.fd_no = ntoh64 (req->fd);
-
-        gf_fd_put (conn->fdtable, state->resolve.fd_no);
-
-        server_release_cbk (frame, NULL, frame->this, 0, 0);
-
-        return 0;
-}
-
-
-static int
-server_fsync_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsync_cbk,
-                    bound_xl, bound_xl->fops->fsync,
-                    state->fd, state->flags);
-        return 0;
-err:
-        server_fsync_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_fsync (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_fsync_req_t  *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->flags         = ntoh32 (req->data);
-
-        gf_resolve_and_resume (frame, server_fsync_resume);
-
-        return 0;
-}
-
-
-
-static int
-server_flush_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_flush_cbk,
-                    bound_xl, bound_xl->fops->flush, state->fd);
-        return 0;
-err:
-        server_flush_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno);
-
-        return 0;
-}
-
-
-static int
-server_flush (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_fsync_req_t  *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-
-        gf_resolve_and_resume (frame, server_flush_resume);
-
-        return 0;
-}
-
-
-
-static int
-server_ftruncate_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_ftruncate_cbk,
-                    bound_xl, bound_xl->fops->ftruncate,
-                    state->fd, state->offset);
-        return 0;
-err:
-        server_ftruncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-static int
-server_ftruncate (call_frame_t *frame, xlator_t *bound_xl,
-                  gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_ftruncate_req_t  *req = NULL;
-        server_state_t          *state = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = ntoh64 (req->fd);
-        state->offset         = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_ftruncate_resume);
-
-        return 0;
-}
-
-
-static int
-server_fstat_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t     *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fstat_cbk,
-                    bound_xl, bound_xl->fops->fstat,
-                    state->fd);
-        return 0;
-err:
-        server_fstat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_fstat (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_fstat_req_t  *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.fd_no   = ntoh64 (req->fd);
-
-        gf_resolve_and_resume (frame, server_fstat_resume);
-
-        return 0;
-}
-
-
-static int
-server_truncate_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_truncate_cbk,
-                    bound_xl, bound_xl->fops->truncate,
-                    &state->loc, state->offset);
-        return 0;
-err:
-        server_truncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-
-static int
-server_truncate (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_truncate_req_t *req = NULL;
-        server_state_t        *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.path  = gf_strdup (req->path);
-        state->resolve.ino   = ntoh64 (req->ino);
-        state->resolve.gen   = ntoh64 (req->gen);
-        state->offset        = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_truncate_resume);
-
-        return 0;
-}
-
-
-static int
-server_unlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_unlink_cbk,
-                    bound_xl, bound_xl->fops->unlink,
-                    &state->loc);
-        return 0;
-err:
-        server_unlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_unlink (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_unlink_req_t *req = NULL;
-        server_state_t      *state = NULL;
-        int                  pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.par    = ntoh64 (req->par);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.bname  = gf_strdup (req->bname + pathlen);
-
-        gf_resolve_and_resume (frame, server_unlink_resume);
-
-        return 0;
-}
-
-
-static int
-server_setxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setxattr_cbk,
-                    bound_xl, bound_xl->fops->setxattr,
-                    &state->loc, state->dict, state->flags);
-        return 0;
-err:
-        server_setxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-
-        return 0;
-}
-
-
-static int
-server_setxattr (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_setxattr_req_t *req = NULL;
-        server_state_t        *state = NULL;
-        dict_t                *dict = NULL;
-        int32_t                ret = -1;
-        size_t                 dict_len = 0;
-        char                  *req_dictbuf = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        dict_len = ntoh32 (req->dict_len);
-
-        state->resolve.type     = RESOLVE_MUST;
-        state->resolve.path     = gf_strdup (req->path + dict_len);
-        state->resolve.ino      = ntoh64 (req->ino);
-        state->resolve.gen      = ntoh64 (req->gen);
-        state->flags            = ntoh32 (req->flags);
-
-        if (dict_len) {
-                req_dictbuf = memdup (req->dict, dict_len);
-
-                dict = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "%"PRId64": %s (%"PRId64"): failed to "
-                                "unserialize request buffer to dictionary",
-                                frame->root->unique, state->loc.path,
-                                state->resolve.ino);
-                        GF_FREE (req_dictbuf);
-                        goto err;
-                }
-
-                dict->extra_free = req_dictbuf;
-                state->dict = dict;
-        }
-
-        gf_resolve_and_resume (frame, server_setxattr_resume);
-
-        return 0;
-err:
-        if (dict)
-                dict_unref (dict);
-
-        server_setxattr_cbk (frame, NULL, frame->this, -1, EINVAL);
-
-        return 0;
-
-}
-
-
-static int
-server_fsetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setxattr_cbk,
-                    bound_xl, bound_xl->fops->fsetxattr,
-                    state->fd, state->dict, state->flags);
-        return 0;
-err:
-        server_fsetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno);
-
-        return 0;
-}
-
-
-static int
-server_fsetxattr (call_frame_t *frame, xlator_t *bound_xl,
-                  gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_fsetxattr_req_t *req = NULL;
-        server_state_t         *state = NULL;
-        dict_t                 *dict = NULL;
-        int32_t                 ret = -1;
-        size_t                  dict_len = 0;
-        char                   *req_dictbuf = NULL;
-
-        req = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        dict_len = ntoh32 (req->dict_len);
-
-        state->resolve.type      = RESOLVE_MUST;
-        state->resolve.fd_no     = ntoh64 (req->fd);
-        state->flags             = ntoh32 (req->flags);
-
-        if (dict_len) {
-                req_dictbuf = memdup (req->dict, dict_len);
-
-                dict = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "%"PRId64": %s (%"PRId64"): failed to "
-                                "unserialize request buffer to dictionary",
-                                frame->root->unique, state->loc.path,
-                                state->resolve.ino);
-                        GF_FREE (req_dictbuf);
-                        goto err;
-                }
-
-                dict->extra_free = req_dictbuf;
-                state->dict = dict;
-        }
-
-        gf_resolve_and_resume (frame, server_fsetxattr_resume);
-
-        return 0;
-err:
-        if (dict)
-                dict_unref (dict);
-
-        server_setxattr_cbk (frame, NULL, frame->this, -1, EINVAL);
-
-        return 0;
-}
-
-
-static int
-server_fxattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fxattrop_cbk,
-                    bound_xl, bound_xl->fops->fxattrop,
-                    state->fd, state->flags, state->dict);
-        return 0;
-err:
-        server_fxattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_fxattrop (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_fxattrop_req_t *req = NULL;
-        dict_t                *dict = NULL;
-        server_state_t        *state = NULL;
-        size_t                 dict_len = 0;
-        char                  *req_dictbuf = NULL;
-        int32_t                ret = -1;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        dict_len = ntoh32 (req->dict_len);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.fd_no   = ntoh64 (req->fd);
-
-        state->resolve.ino     = ntoh64 (req->ino);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->flags           = ntoh32 (req->flags);
-
-        if (dict_len) {
-                /* Unserialize the dictionary */
-                req_dictbuf = memdup (req->dict, dict_len);
-
-                dict = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "fd - %"PRId64" (%"PRId64"): failed to unserialize "
-                                "request buffer to dictionary",
-                                state->resolve.fd_no, state->fd->inode->ino);
-                        GF_FREE (req_dictbuf);
-                        goto fail;
-                }
-                dict->extra_free = req_dictbuf;
-                state->dict = dict;
-                dict = NULL;
-        }
-
-        gf_resolve_and_resume (frame, server_fxattrop_resume);
-
-        return 0;
-
-fail:
-        if (dict)
-                dict_unref (dict);
-
-        server_fxattrop_cbk (frame, NULL, frame->this, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-static int
-server_xattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_xattrop_cbk,
-                    bound_xl, bound_xl->fops->xattrop,
-                    &state->loc, state->flags, state->dict);
-        return 0;
-err:
-        server_xattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_xattrop (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_xattrop_req_t  *req = NULL;
-        dict_t                *dict = NULL;
-        server_state_t        *state = NULL;
-        size_t                 dict_len = 0;
-        char                  *req_dictbuf = NULL;
-        int32_t                ret = -1;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        dict_len = ntoh32 (req->dict_len);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.path    = gf_strdup (req->path + dict_len);
-        state->resolve.ino     = ntoh64 (req->ino);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->flags           = ntoh32 (req->flags);
-
-        if (dict_len) {
-                /* Unserialize the dictionary */
-                req_dictbuf = memdup (req->dict, dict_len);
-
-                dict = dict_new ();
-
-                ret = dict_unserialize (req_dictbuf, dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (bound_xl->name, GF_LOG_ERROR,
-                                "fd - %"PRId64" (%"PRId64"): failed to unserialize "
-                                "request buffer to dictionary",
-                                state->resolve.fd_no, state->fd->inode->ino);
-                        GF_FREE (req_dictbuf);
-                        goto fail;
-                }
-                dict->extra_free = req_dictbuf;
-                state->dict = dict;
-                dict = NULL;
-        }
-
-        gf_resolve_and_resume (frame, server_xattrop_resume);
-
-        return 0;
-
-fail:
-        if (dict)
-                dict_unref (dict);
-
-        server_xattrop_cbk (frame, NULL, frame->this, -1, EINVAL, NULL);
-        return 0;
-}
-
-
-static int
-server_getxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_getxattr_cbk,
-                    bound_xl, bound_xl->fops->getxattr,
-                    &state->loc, state->name);
-        return 0;
-err:
-        server_getxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_getxattr (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_getxattr_req_t  *req = NULL;
-        server_state_t         *state = NULL;
-        size_t                  namelen = 0;
-        size_t                  pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.path  = gf_strdup (req->path);
-        state->resolve.ino   = ntoh64 (req->ino);
-        state->resolve.gen   = ntoh64 (req->gen);
-
-        namelen = ntoh32 (req->namelen);
-        if (namelen)
-                state->name = gf_strdup (req->name + pathlen);
-
-        gf_resolve_and_resume (frame, server_getxattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_fgetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fgetxattr_cbk,
-                    bound_xl, bound_xl->fops->fgetxattr,
-                    state->fd, state->name);
-        return 0;
-err:
-        server_fgetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_fgetxattr (call_frame_t *frame, xlator_t *bound_xl,
-                  gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_fgetxattr_req_t *req = NULL;
-        server_state_t         *state = NULL;
-        size_t                  namelen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-
-        namelen = ntoh32 (req->namelen);
-        if (namelen)
-                state->name = gf_strdup (req->name);
-
-        gf_resolve_and_resume (frame, server_fgetxattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_removexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_removexattr_cbk,
-                    bound_xl, bound_xl->fops->removexattr,
-                    &state->loc, state->name);
-        return 0;
-err:
-        server_removexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                                state->resolve.op_errno);
-        return 0;
-}
-
-
-static int
-server_removexattr (call_frame_t *frame, xlator_t *bound_xl,
-                    gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_fop_removexattr_req_t  *req = NULL;
-        server_state_t            *state = NULL;
-        size_t                     pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.ino    = ntoh64 (req->ino);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->name           = gf_strdup (req->name + pathlen);
-
-        gf_resolve_and_resume (frame, server_removexattr_resume);
-
-        return 0;
-}
-
-
-static int
-server_statfs_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t      *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret !=0)
-                goto err;
-
-        STACK_WIND (frame, server_statfs_cbk,
-                    bound_xl, bound_xl->fops->statfs,
-                    &state->loc);
-        return 0;
-
-err:
-        server_statfs_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_statfs (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_statfs_req_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req = gf_param (hdr);
-
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.ino    = ntoh64 (req->ino);
-        if (!state->resolve.ino)
-                state->resolve.ino = 1;
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-
-        gf_resolve_and_resume (frame, server_statfs_resume);
-
-        return 0;
-}
-
-
-static int
-server_opendir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-
-        STACK_WIND (frame, server_opendir_cbk,
-                    bound_xl, bound_xl->fops->opendir,
-                    &state->loc, state->fd);
-        return 0;
-err:
-        server_opendir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-static int
-server_opendir (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_opendir_req_t  *req = NULL;
-        server_state_t        *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.ino    = ntoh64 (req->ino);
-        state->resolve.gen    = ntoh64 (req->gen);
-
-        gf_resolve_and_resume (frame, server_opendir_resume);
-
-        return 0;
-}
-
-
-static int
-server_releasedir (call_frame_t *frame, xlator_t *bound_xl,
-                   gf_hdr_common_t *hdr, size_t hdrlen,
-                   struct iobuf *iobuf)
-{
-        gf_cbk_releasedir_req_t *req = NULL;
-        server_connection_t     *conn = NULL;
-        uint64_t                 fd_no = 0;
-
-        conn = SERVER_CONNECTION (frame);
-
-        req = gf_param (hdr);
-
-        fd_no = ntoh64 (req->fd);
-
-        gf_fd_put (conn->fdtable, fd_no);
-
-        server_releasedir_cbk (frame, NULL, frame->this, 0, 0);
-
-        return 0;
-}
-
-/*
- * server_readdirp_cbk - getdents callback for server protocol
- * @frame: call frame
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- *
- * not for external reference
- */
-static int
-server_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        gf_hdr_common_t         *hdr = NULL;
-        gf_fop_readdirp_rsp_t   *rsp = NULL;
-        size_t                  hdrlen = 0;
-        size_t                  buf_size = 0;
-        int32_t                 gf_errno = 0;
-        server_state_t          *state = NULL;
-
-        if (op_ret > 0)
-                buf_size = gf_dirent_serialize (entries, NULL, 0);
-
-        hdrlen = gf_hdr_len (rsp, buf_size);
-        hdr    = gf_hdr_new (rsp, buf_size);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret > 0) {
-                rsp->size = hton32 (buf_size);
-                gf_dirent_serialize (entries, rsp->buf, buf_size);
-        } else {
-                state = CALL_STATE(frame);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%"PRId64": READDIRP %"PRId64" (%"PRId64") ==>"
-                        "%"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? state->fd->inode->ino : 0, op_ret,
-                        strerror (op_errno));
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_READDIRP,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-static int
-server_readdirp_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readdirp_cbk, bound_xl,
-                    bound_xl->fops->readdirp, state->fd, state->size,
-                    state->offset);
-
-        return 0;
-err:
-        server_readdirp_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-/*
- * server_readdirp - readdirp function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- * not for external reference
- */
-static int
-server_readdirp (call_frame_t *frame, xlator_t *bound_xl, gf_hdr_common_t *hdr,
-                 size_t hdrlen, struct iobuf *iobuf)
-{
-        gf_fop_readdirp_req_t *req = NULL;
-        server_state_t *state = NULL;
-        server_connection_t *conn = NULL;
-
-        conn = SERVER_CONNECTION(frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->size   = ntoh32 (req->size);
-        state->offset = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_readdirp_resume);
-
-        return 0;
-}
-
-
-static int
- server_readdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readdir_cbk,
-                    bound_xl,
-                    bound_xl->fops->readdir,
-                    state->fd, state->size, state->offset);
-
-        return 0;
-err:
-        server_readdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-/*
- * server_readdir - readdir function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- * not for external reference
- */
-static int
-server_readdir (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_readdir_req_t *req = NULL;
-        server_state_t *state = NULL;
-        server_connection_t *conn = NULL;
-
-        conn = SERVER_CONNECTION(frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->size   = ntoh32 (req->size);
-        state->offset = ntoh64 (req->offset);
-
-        gf_resolve_and_resume (frame, server_readdir_resume);
-
-        return 0;
-}
-
-static int
-server_fsyncdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsyncdir_cbk,
-                    bound_xl,
-                    bound_xl->fops->fsyncdir,
-                    state->fd, state->flags);
-        return 0;
-
-err:
-        server_fsyncdir_cbk (frame, NULL, frame->this,
-                             state->resolve.op_ret,
-                             state->resolve.op_errno);
-        return 0;
-}
-
-/*
- * server_fsyncdir - fsyncdir function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- * not for external reference
- */
-static int
-server_fsyncdir (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_fsyncdir_req_t *req = NULL;
-        server_state_t        *state = NULL;
-        server_connection_t   *conn = NULL;
-
-        conn = SERVER_CONNECTION (frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->flags = ntoh32 (req->data);
-
-        gf_resolve_and_resume (frame, server_fsyncdir_resume);
-
-        return 0;
-}
-
-
-static int
-server_mknod_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_mknod_cbk,
-                    bound_xl, bound_xl->fops->mknod,
-                    &(state->loc), state->mode, state->dev,
-                    state->params);
-
-        return 0;
-err:
-        server_mknod_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-
-static int
-server_mknod (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_mknod_req_t *req = NULL;
-        server_state_t     *state = NULL;
-        size_t              pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type    = RESOLVE_NOT;
-        state->resolve.par     = ntoh64 (req->par);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->resolve.path    = gf_strdup (req->path);
-        state->resolve.bname   = gf_strdup (req->bname + pathlen);
-
-        state->mode = ntoh32 (req->mode);
-        state->dev  = ntoh64 (req->dev);
-
-        gf_resolve_and_resume (frame, server_mknod_resume);
-
-        return 0;
-}
-
-
-static int
-server_mkdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_mkdir_cbk,
-                    bound_xl, bound_xl->fops->mkdir,
-                    &(state->loc), state->mode,
-                    state->params);
-
-        return 0;
-err:
-        server_mkdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_mkdir (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_mkdir_req_t *req = NULL;
-        server_state_t     *state = NULL;
-        size_t              pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type    = RESOLVE_NOT;
-        state->resolve.par     = ntoh64 (req->par);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->resolve.path    = gf_strdup (req->path);
-        state->resolve.bname   = gf_strdup (req->bname + pathlen);
-
-        state->mode = ntoh32 (req->mode);
-
-        gf_resolve_and_resume (frame, server_mkdir_resume);
-
-        return 0;
-}
-
-
-static int
-server_rmdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_rmdir_cbk,
-                    bound_xl, bound_xl->fops->rmdir, &state->loc);
-        return 0;
-err:
-        server_rmdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-static int
-server_rmdir (call_frame_t *frame, xlator_t *bound_xl,
-              gf_hdr_common_t *hdr, size_t hdrlen,
-              struct iobuf *iobuf)
-{
-        gf_fop_rmdir_req_t *req = NULL;
-        server_state_t     *state = NULL;
-        int                 pathlen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.par     = ntoh64 (req->par);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->resolve.path    = gf_strdup (req->path);
-        state->resolve.bname   = gf_strdup (req->bname + pathlen);
-
-        gf_resolve_and_resume (frame, server_rmdir_resume);
-
-        return 0;
-}
-
-
-static int
-server_inodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_inodelk_cbk,
-                    bound_xl, bound_xl->fops->inodelk,
-                    state->volume, &state->loc, state->cmd, &state->flock);
-        return 0;
-err:
-        server_inodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno);
-        return 0;
-}
-
-
-static int
-server_inodelk (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_inodelk_req_t *req = NULL;
-        server_state_t       *state = NULL;
-        size_t                pathlen = 0;
-        size_t                vollen  = 0;
-        int                   cmd = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-        vollen  = STRLEN_0 (req->volume + pathlen);
-
-        state->resolve.type    = RESOLVE_EXACT;
-        state->resolve.ino     = ntoh64 (req->ino);
-        state->resolve.gen     = ntoh64 (req->gen);
-        state->resolve.path    = gf_strdup (req->path);
-
-        cmd = ntoh32 (req->cmd);
-        switch (cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        }
-
-        state->type = ntoh32 (req->type);
-        state->volume = gf_strdup (req->volume + pathlen);
-
-        gf_flock_to_flock (&req->flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        }
-
-        gf_resolve_and_resume (frame, server_inodelk_resume);
-
-        return 0;
-}
-
-static int
-server_finodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_finodelk_cbk,
-                    BOUND_XL(frame),
-                    BOUND_XL(frame)->fops->finodelk,
-                    state->volume, state->fd, state->cmd, &state->flock);
-
-        return 0;
-err:
-        server_finodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-
-        return 0;
-}
-
-static int
-server_finodelk (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_finodelk_req_t  *req = NULL;
-        server_state_t         *state = NULL;
-        server_connection_t    *conn = NULL;
-
-        conn = SERVER_CONNECTION(frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_EXACT;
-        state->volume = gf_strdup (req->volume);
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->cmd = ntoh32 (req->cmd);
-
-        switch (state->cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        }
-
-        state->type = ntoh32 (req->type);
-
-        gf_flock_to_flock (&req->flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        }
-
-        gf_resolve_and_resume (frame, server_finodelk_resume);
-
-        return 0;
-}
-
-
-static int
-server_entrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_entrylk_cbk,
-                    bound_xl, bound_xl->fops->entrylk,
-                    state->volume, &state->loc, state->name,
-                    state->cmd, state->type);
-        return 0;
-err:
-        server_entrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno);
-        return 0;
-}
-
-
-static int
-server_entrylk (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        gf_fop_entrylk_req_t *req = NULL;
-        server_state_t       *state = NULL;
-        size_t                pathlen = 0;
-        size_t                namelen = 0;
-        size_t                vollen  = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-        namelen = ntoh64 (req->namelen);
-        vollen = STRLEN_0(req->volume + pathlen + namelen);
-
-        state->resolve.type   = RESOLVE_EXACT;
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.ino    = ntoh64 (req->ino);
-        state->resolve.gen    = ntoh64 (req->gen);
-
-        if (namelen)
-                state->name   = gf_strdup (req->name + pathlen);
-        state->volume         = gf_strdup (req->volume + pathlen + namelen);
-
-        state->cmd            = ntoh32 (req->cmd);
-        state->type           = ntoh32 (req->type);
-
-        gf_resolve_and_resume (frame, server_entrylk_resume);
-
-        return 0;
-}
-
-static int
-server_fentrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fentrylk_cbk,
-                    BOUND_XL(frame),
-                    BOUND_XL(frame)->fops->fentrylk,
-                    state->volume, state->fd, state->name,
-                    state->cmd, state->type);
-
-        return 0;
-err:
-        server_fentrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-        return 0;
-}
-
-static int
-server_fentrylk (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_fop_fentrylk_req_t *req = NULL;
-        server_state_t        *state = NULL;
-        size_t                 namelen = 0;
-        size_t                 vollen  = 0;
-        server_connection_t   *conn = NULL;
-
-        conn = SERVER_CONNECTION (frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE(frame);
-        vollen = STRLEN_0(req->volume + namelen);
-
-        state->resolve.type = RESOLVE_EXACT;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->cmd  = ntoh32 (req->cmd);
-        state->type = ntoh32 (req->type);
-        namelen = ntoh64 (req->namelen);
-        if (namelen)
-                state->name = req->name;
-        state->volume = gf_strdup (req->volume + namelen);
-
-
-        gf_resolve_and_resume (frame, server_fentrylk_resume);
-
-        return 0;
-}
-
-
-static int
-server_access_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_access_cbk,
-                    bound_xl, bound_xl->fops->access,
-                    &state->loc, state->mask);
-        return 0;
-err:
-        server_access_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno);
-        return 0;
-}
-
-
-static int
-server_access (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_access_req_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.ino   = hton64 (req->ino);
-        state->resolve.gen   = hton64 (req->gen);
-        state->resolve.path  = gf_strdup (req->path);
-
-        state->mask  = ntoh32 (req->mask);
-
-        gf_resolve_and_resume (frame, server_access_resume);
-
-        return 0;
-}
-
-
-static int
-server_symlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_symlink_cbk,
-                    bound_xl, bound_xl->fops->symlink,
-                    state->name, &state->loc, state->params);
-
-        return 0;
-err:
-        server_symlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-
-static int
-server_symlink (call_frame_t *frame, xlator_t *bound_xl,
-                gf_hdr_common_t *hdr, size_t hdrlen,
-                struct iobuf *iobuf)
-{
-        server_state_t       *state = NULL;
-        gf_fop_symlink_req_t *req = NULL;
-        size_t                pathlen = 0;
-        size_t                baselen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        pathlen = STRLEN_0 (req->path);
-        baselen = STRLEN_0 (req->bname + pathlen);
-
-        state->resolve.type   = RESOLVE_NOT;
-        state->resolve.par    = ntoh64 (req->par);
-        state->resolve.gen    = ntoh64 (req->gen);
-        state->resolve.path   = gf_strdup (req->path);
-        state->resolve.bname  = gf_strdup (req->bname + pathlen);
-        state->name           = gf_strdup (req->linkname + pathlen + baselen);
-
-        gf_resolve_and_resume (frame, server_symlink_resume);
-
-        return 0;
-}
-
-
-static int
-server_link_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        if (state->resolve2.op_ret != 0) {
-                op_ret   = state->resolve2.op_ret;
-                op_errno = state->resolve2.op_errno;
-                goto err;
-        }
-
-        state->loc2.inode = inode_ref (state->loc.inode);
-
-        STACK_WIND (frame, server_link_cbk,
-                    bound_xl, bound_xl->fops->link,
-                    &state->loc, &state->loc2);
-        return 0;
-err:
-        server_link_cbk (frame, NULL, frame->this, op_ret, op_errno,
-                         NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_link (call_frame_t *frame, xlator_t *this,
-             gf_hdr_common_t *hdr, size_t hdrlen,
-             struct iobuf *iobuf)
-{
-        gf_fop_link_req_t *req = NULL;
-        server_state_t    *state = NULL;
-        size_t             oldpathlen = 0;
-        size_t             newpathlen = 0;
-        size_t             newbaselen = 0;
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-        oldpathlen = STRLEN_0 (req->oldpath);
-        newpathlen = STRLEN_0 (req->newpath  + oldpathlen);
-        newbaselen = STRLEN_0 (req->newbname + oldpathlen + newpathlen);
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.path    = gf_strdup (req->oldpath);
-        state->resolve.ino     = ntoh64 (req->oldino);
-        state->resolve.gen     = ntoh64 (req->oldgen);
-
-        state->resolve2.type   = RESOLVE_NOT;
-        state->resolve2.path   = gf_strdup (req->newpath + oldpathlen);
-        state->resolve2.bname  = gf_strdup (req->newbname + oldpathlen + newpathlen);
-        state->resolve2.par    = ntoh64 (req->newpar);
-        state->resolve2.gen    = ntoh64 (req->newgen);
-
-        gf_resolve_and_resume (frame, server_link_resume);
-
-        return 0;
-}
-
-
-static int
-server_rename_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        if (state->resolve2.op_ret != 0) {
-                op_ret   = state->resolve2.op_ret;
-                op_errno = state->resolve2.op_errno;
-                goto err;
-        }
-
-        STACK_WIND (frame, server_rename_cbk,
-                    bound_xl, bound_xl->fops->rename,
-                    &state->loc, &state->loc2);
-        return 0;
-err:
-        server_rename_cbk (frame, NULL, frame->this, op_ret, op_errno,
-                           NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-static int
-server_rename (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        gf_fop_rename_req_t *req = NULL;
-        server_state_t      *state = NULL;
-        size_t               oldpathlen = 0;
-        size_t               oldbaselen = 0;
-        size_t               newpathlen = 0;
-        size_t               newbaselen = 0;
-
-        req = gf_param (hdr);
-
-        state = CALL_STATE (frame);
-        oldpathlen = STRLEN_0 (req->oldpath);
-        oldbaselen = STRLEN_0 (req->oldbname + oldpathlen);
-        newpathlen = STRLEN_0 (req->newpath  + oldpathlen + oldbaselen);
-        newbaselen = STRLEN_0 (req->newbname + oldpathlen +
-                               oldbaselen + newpathlen);
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.path   = gf_strdup (req->oldpath);
-        state->resolve.bname  = gf_strdup (req->oldbname + oldpathlen);
-        state->resolve.par    = ntoh64 (req->oldpar);
-        state->resolve.gen    = ntoh64 (req->oldgen);
-
-        state->resolve2.type  = RESOLVE_MAY;
-        state->resolve2.path  = gf_strdup (req->newpath  + oldpathlen + oldbaselen);
-        state->resolve2.bname = gf_strdup (req->newbname + oldpathlen + oldbaselen +
-                                        newpathlen);
-        state->resolve2.par   = ntoh64 (req->newpar);
-        state->resolve2.gen   = ntoh64 (req->newgen);
-
-        gf_resolve_and_resume (frame, server_rename_resume);
-
-        return 0;
-}
-
-static int
-server_lk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_lk_cbk,
-                    BOUND_XL(frame),
-                    BOUND_XL(frame)->fops->lk,
-                    state->fd, state->cmd, &state->flock);
-
-        return 0;
-
-err:
-        server_lk_cbk (frame, NULL, frame->this,
-                       state->resolve.op_ret,
-                       state->resolve.op_errno,
-                       NULL);
-        return 0;
-}
-
-/*
- * server_lk - lk function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- * not for external reference
- */
-
-static int
-server_lk (call_frame_t *frame, xlator_t *bound_xl,
-           gf_hdr_common_t *hdr, size_t hdrlen,
-           struct iobuf *iobuf)
-{
-        gf_fop_lk_req_t     *req = NULL;
-        server_state_t      *state = NULL;
-        server_connection_t *conn = NULL;
-
-        conn = SERVER_CONNECTION (frame);
-
-        req   = gf_param (hdr);
-        state = CALL_STATE (frame);
-
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->cmd =  ntoh32 (req->cmd);
-        state->type = ntoh32 (req->type);
-
-        switch (state->cmd) {
-        case GF_LK_GETLK:
-                state->cmd = F_GETLK;
-                break;
-        case GF_LK_SETLK:
-                state->cmd = F_SETLK;
-                break;
-        case GF_LK_SETLKW:
-                state->cmd = F_SETLKW;
-                break;
-        }
-
-        gf_flock_to_flock (&req->flock, &state->flock);
-
-        switch (state->type) {
-        case GF_LK_F_RDLCK:
-                state->flock.l_type = F_RDLCK;
-                break;
-        case GF_LK_F_WRLCK:
-                state->flock.l_type = F_WRLCK;
-                break;
-        case GF_LK_F_UNLCK:
-                state->flock.l_type = F_UNLCK;
-                break;
-        default:
-                gf_log (bound_xl->name, GF_LOG_ERROR,
-                        "fd - %"PRId64" (%"PRId64"): Unknown lock type: %"PRId32"!",
-                        state->resolve.fd_no, state->fd->inode->ino, state->type);
-                break;
-        }
-
-
-        gf_resolve_and_resume (frame, server_lk_resume);
-
-        return 0;
-}
-
-/* xxx_MOPS */
-static int
-_volfile_update_checksum (xlator_t *this, char *key, uint32_t checksum)
-{
-        server_conf_t       *conf         = NULL;
-        struct _volfile_ctx *temp_volfile = NULL;
-
-        conf         = this->private;
-        temp_volfile = conf->volfile;
-
-        while (temp_volfile) {
-                if ((NULL == key) && (NULL == temp_volfile->key))
-                        break;
-                if ((NULL == key) || (NULL == temp_volfile->key)) {
-                        temp_volfile = temp_volfile->next;
-                        continue;
-                }
-                if (strcmp (temp_volfile->key, key) == 0)
-                        break;
-                temp_volfile = temp_volfile->next;
-        }
-
-        if (!temp_volfile) {
-                temp_volfile = GF_CALLOC (1, sizeof (struct _volfile_ctx),
-                                          gf_server_mt_volfile_ctx);
-                if (!temp_volfile)
-                        goto out;
-                temp_volfile->next  = conf->volfile;
-                temp_volfile->key   = (key)? gf_strdup (key): NULL;
-                temp_volfile->checksum = checksum;
-
-                conf->volfile = temp_volfile;
-                goto out;
-        }
-
-        if (temp_volfile->checksum != checksum) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "the volume file got modified between earlier access "
-                        "and now, this may lead to inconsistency between "
-                        "clients, advised to remount client");
-                temp_volfile->checksum  = checksum;
-        }
-
- out:
-        return 0;
-}
-
-
-size_t
-build_volfile_path (xlator_t *this, const char *key, char *path,
-                    size_t path_len)
-{
-        int             ret = -1;
-        int             free_filename = 0;
-        int             free_conf_dir = 0;
-        char            *filename = NULL;
-        char            *conf_dir = CONFDIR;
-        struct stat      buf      = {0,};
-        data_t *         conf_dir_data = NULL;
-        char             data_key[256] = {0,};
-
-        /* Inform users that this option is changed now */
-        ret = dict_get_str (this->options, "client-volume-filename",
-                            &filename);
-        if (ret == 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "option 'client-volume-filename' is changed to "
-                        "'volume-filename.<key>' which now takes 'key' as an "
-                        "option to choose/fetch different files from server. "
-                        "Refer documentation or contact developers for more "
-                        "info. Currently defaulting to given file '%s'",
-                        filename);
-        }
-
-        if (key && !filename) {
-                sprintf (data_key, "volume-filename.%s", key);
-                ret = dict_get_str (this->options, data_key, &filename);
-
-                if (ret < 0) {
-
-                        conf_dir_data = dict_get (this->options, "conf-dir");
-                        if (conf_dir_data) {
-                                /* Check whether the specified directory exists,
-                                   or directory specified is non standard */
-                                ret = stat (conf_dir_data->data, &buf);
-                                if ((ret != 0) || !S_ISDIR (buf.st_mode)) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Directory '%s' doesn't"
-                                                "exist, exiting.",
-                                                conf_dir_data->data);
-                                        ret = -1;
-                                        goto out;
-                                }
-                                /* Make sure that conf-dir doesn't
-                                 * contain ".." in path
-                                 */
-                                if ((gf_strstr (conf_dir_data->data,
-                                                "/", "..")) == -1) {
-                                        ret = -1;
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: invalid conf_dir",
-                                                conf_dir_data->data);
-                                        goto out;
-                                }
-
-                                /* Make sure that key doesn't
-                                 * contain "../" in path
-                                 */
-
-                                if ((gf_strstr (key, "/", "..")) == -1) {
-                                        ret = -1;
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: invalid key", key);
-                                        goto out;
-                                }
-
-                                conf_dir = gf_strdup (conf_dir_data->data);
-                                free_conf_dir = 1;
-                        }
-
-                        ret = gf_asprintf (&filename, "%s/%s.vol",
-                                           conf_dir, key);
-                        if (-1 == ret)
-                                goto out;
-
-                        free_filename = 1;
-                }
-        }
-
-        if (!filename) {
-                ret = dict_get_str (this->options,
-                                    "volume-filename.default", &filename);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no default volume filename given, "
-                                "defaulting to %s", DEFAULT_VOLUME_FILE_PATH);
-                        filename = DEFAULT_VOLUME_FILE_PATH;
-                }
-        }
-
-        ret = -1;
-
-        if ((filename) && (path_len > strlen (filename))) {
-                strcpy (path, filename);
-                ret = strlen (filename);
-        }
-
-out:
-        if (free_conf_dir)
-                GF_FREE (conf_dir);
-
-        if (free_filename)
-                GF_FREE (filename);
-
-        return ret;
-}
-
-static int
-_validate_volfile_checksum (xlator_t *this, char *key,
-                            uint32_t checksum)
-{
-        char                 filename[ZR_PATH_MAX] = {0,};
-        server_conf_t       *conf         = NULL;
-        struct _volfile_ctx *temp_volfile = NULL;
-        int                  ret          = 0;
-        uint32_t             local_checksum = 0;
-
-        conf         = this->private;
-        temp_volfile = conf->volfile;
-
-        if (!checksum)
-                goto out;
-
-        if (!temp_volfile) {
-                ret = build_volfile_path (this, key, filename,
-                                          sizeof (filename));
-                if (ret <= 0)
-                        goto out;
-                ret = open (filename, O_RDONLY);
-                if (-1 == ret) {
-                        ret = 0;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to open volume file (%s) : %s",
-                                filename, strerror (errno));
-                        goto out;
-                }
-                get_checksum_for_file (ret, &local_checksum);
-                _volfile_update_checksum (this, key, local_checksum);
-                close (ret);
-        }
-
-        temp_volfile = conf->volfile;
-        while (temp_volfile) {
-                if ((NULL == key) && (NULL == temp_volfile->key))
-                        break;
-                if ((NULL == key) || (NULL == temp_volfile->key)) {
-                        temp_volfile = temp_volfile->next;
-                        continue;
-                }
-                if (strcmp (temp_volfile->key, key) == 0)
-                        break;
-                temp_volfile = temp_volfile->next;
-        }
-
-        if (!temp_volfile)
-                goto out;
-
-        if ((temp_volfile->checksum) &&
-            (checksum != temp_volfile->checksum))
-                ret = -1;
-
-out:
-        return ret;
-}
-
-/* Management Calls */
-/*
- * mop_getspec - getspec function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params:
- *
- */
-static int
-mop_getspec (call_frame_t *frame, xlator_t *bound_xl,
-             gf_hdr_common_t *hdr, size_t hdrlen,
-             struct iobuf *iobuf)
-{
-        gf_hdr_common_t      *_hdr = NULL;
-        gf_mop_getspec_rsp_t *rsp = NULL;
-        int32_t               ret = -1;
-        int32_t               op_errno = ENOENT;
-        int32_t               gf_errno = 0;
-        int32_t               spec_fd = -1;
-        size_t                file_len = 0;
-        size_t                _hdrlen = 0;
-        char                  filename[ZR_PATH_MAX] = {0,};
-        struct stat           stbuf = {0,};
-        gf_mop_getspec_req_t *req = NULL;
-        uint32_t              checksum = 0;
-        uint32_t              flags  = 0;
-        uint32_t              keylen = 0;
-        char                 *key = NULL;
-        server_conf_t        *conf = NULL;
-
-        req   = gf_param (hdr);
-        flags = ntoh32 (req->flags);
-        keylen = ntoh32 (req->keylen);
-        if (keylen) {
-                key = req->key;
-        }
-
-        conf = frame->this->private;
-
-        ret = build_volfile_path (frame->this, key, filename,
-                                  sizeof (filename));
-        if (ret > 0) {
-                /* to allocate the proper buffer to hold the file data */
-                ret = stat (filename, &stbuf);
-                if (ret < 0){
-                        gf_log (frame->this->name, GF_LOG_ERROR,
-                                "Unable to stat %s (%s)",
-                                filename, strerror (errno));
-                        goto fail;
-                }
-
-                spec_fd = open (filename, O_RDONLY);
-                if (spec_fd < 0) {
-                        gf_log (frame->this->name, GF_LOG_ERROR,
-                                "Unable to open %s (%s)",
-                                filename, strerror (errno));
-                        goto fail;
-                }
-                ret = 0;
-                file_len = stbuf.st_size;
-                if (conf->verify_volfile_checksum) {
-                        get_checksum_for_file (spec_fd, &checksum);
-                        _volfile_update_checksum (frame->this, key, checksum);
-                }
-        } else {
-                errno = ENOENT;
-        }
-
-fail:
-        op_errno = errno;
-
-        _hdrlen = gf_hdr_len (rsp, file_len + 1);
-        _hdr    = gf_hdr_new (rsp, file_len + 1);
-        rsp     = gf_param (_hdr);
-
-        _hdr->rsp.op_ret = hton32 (ret);
-        gf_errno         = gf_errno_to_error (op_errno);
-        _hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (file_len) {
-                ret = read (spec_fd, rsp->spec, file_len);
-                close (spec_fd);
-        }
-        protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_GETSPEC,
-                               _hdr, _hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-
-static int
-server_checksum (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_CHECKSUM,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-server_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
-{
-        gf_hdr_common_t       *hdr = NULL;
-        gf_fop_rchecksum_rsp_t *rsp = NULL;
-        size_t                 hdrlen = 0;
-        int32_t                gf_errno = 0;
-
-        hdrlen = gf_hdr_len (rsp, MD5_DIGEST_LEN + 1);
-        hdr    = gf_hdr_new (rsp, MD5_DIGEST_LEN + 1);
-        rsp    = gf_param (hdr);
-
-        hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno        = gf_errno_to_error (op_errno);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-
-        if (op_ret >= 0) {
-                rsp->weak_checksum = weak_checksum;
-
-                memcpy (rsp->strong_checksum,
-                        strong_checksum, MD5_DIGEST_LEN);
-
-                rsp->strong_checksum[MD5_DIGEST_LEN] = '\0';
-        }
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_RCHECKSUM,
-                               hdr, hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-static int
-server_rchecksum_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        STACK_WIND (frame, server_rchecksum_cbk,
-                    bound_xl,
-                    bound_xl->fops->rchecksum,
-                    state->fd, state->offset, state->size);
-
-        return 0;
-err:
-        server_rchecksum_cbk (frame, NULL, frame->this, -1, EINVAL, 0, NULL);
-
-        return 0;
-
-}
-
-static int
-server_rchecksum (call_frame_t *frame, xlator_t *bound_xl,
-                  gf_hdr_common_t *hdr, size_t hdrlen,
-                  struct iobuf *iobuf)
-{
-        gf_fop_rchecksum_req_t  *req = NULL;
-        server_state_t          *state = NULL;
-        server_connection_t     *conn = NULL;
-
-        conn = SERVER_CONNECTION(frame);
-
-        req = gf_param (hdr);
-
-        state = CALL_STATE(frame);
-
-        state->resolve.type = RESOLVE_MAY;
-        state->resolve.fd_no = ntoh64 (req->fd);
-        state->offset = ntoh64 (req->offset);
-        state->size   = ntoh32 (req->len);
-
-        gf_resolve_and_resume (frame, server_rchecksum_resume);
-
-        return 0;
-}
-
-
-/*
- * mop_unlock - unlock management function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- */
-static int
-mop_getvolume (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *hdr, size_t hdrlen,
-               struct iobuf *iobuf)
-{
-        return 0;
-}
-
-struct __get_xl_struct {
-        const char *name;
-        xlator_t *reply;
-};
-
-static void __check_and_set (xlator_t *each, void *data)
-{
-        if (!strcmp (each->name,
-                     ((struct __get_xl_struct *) data)->name))
-                ((struct __get_xl_struct *) data)->reply = each;
-}
-
-static xlator_t *
-get_xlator_by_name (xlator_t *some_xl, const char *name)
-{
-        struct __get_xl_struct get = {
-                .name = name,
-                .reply = NULL
-        };
-
-        xlator_foreach (some_xl, __check_and_set, &get);
-
-        return get.reply;
-}
-
-
-/*
- * mop_setvolume - setvolume management function for server protocol
- * @frame: call frame
- * @bound_xl:
- * @params: parameter dictionary
- *
- */
-static int
-mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
-               gf_hdr_common_t *req_hdr, size_t req_hdrlen,
-               struct iobuf *iobuf)
-{
-        server_connection_t         *conn = NULL;
-        server_conf_t               *conf = NULL;
-        gf_hdr_common_t             *rsp_hdr = NULL;
-        gf_mop_setvolume_req_t      *req = NULL;
-        gf_mop_setvolume_rsp_t      *rsp = NULL;
-        peer_info_t                 *peerinfo = NULL;
-        int32_t                      ret = -1;
-        int32_t                      op_ret = -1;
-        int32_t                      op_errno = EINVAL;
-        int32_t                      gf_errno = 0;
-        dict_t                      *reply = NULL;
-        dict_t                      *config_params = NULL;
-        dict_t                      *params = NULL;
-        char                        *name = NULL;
-        char                        *version = NULL;
-        char                        *process_uuid = NULL;
-        xlator_t                    *xl = NULL;
-        transport_t                 *trans = NULL;
-        size_t                       rsp_hdrlen = -1;
-        size_t                       dict_len = -1;
-        size_t                       req_dictlen = -1;
-        char                        *msg = NULL;
-        char                        *volfile_key = NULL;
-        uint32_t                     checksum = 0;
-        int32_t                      lru_limit = 1024;
-
-        params = dict_new ();
-        reply  = dict_new ();
-
-        req    = gf_param (req_hdr);
-        req_dictlen = ntoh32 (req->dict_len);
-        ret = dict_unserialize (req->buf, req_dictlen, &params);
-
-        config_params = dict_copy_with_ref (frame->this->options, NULL);
-        trans         = TRANSPORT_FROM_FRAME(frame);
-        conf          = SERVER_CONF(frame);
-
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "Internal error: failed to unserialize "
-                                    "request dictionary");
-                if (ret < 0)
-                        gf_log (bound_xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg \"%s\"",
-                                "Internal error: failed to unserialize "
-                                "request dictionary");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        ret = dict_get_str (params, "process-uuid", &process_uuid);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "UUID not specified");
-                if (ret < 0)
-                        gf_log (bound_xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-
-        conn = gf_server_connection_get (frame->this, process_uuid);
-        if (trans->xl_private != conn)
-                trans->xl_private = conn;
-
-        ret = dict_get_str (params, "protocol-version", &version);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "No version number specified");
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        ret = strcmp (version, GF_PROTOCOL_VERSION);
-        if (ret != 0) {
-                ret = gf_asprintf (&msg, "protocol version mismatch: client(%s) "
-                                "- server(%s)", version, GF_PROTOCOL_VERSION);
-                if (-1 == ret) {
-                        gf_log (trans->xl->name, GF_LOG_ERROR,
-                                "gf_asprintf failed while setting up error msg");
-                        goto fail;
-                }
-                ret = dict_set_dynstr (reply, "ERROR", msg);
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        ret = dict_get_str (params,
-                            "remote-subvolume", &name);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "No remote-subvolume option specified");
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        xl = get_xlator_by_name (frame->this, name);
-        if (xl == NULL) {
-                ret = gf_asprintf (&msg, "remote-subvolume \"%s\" is not found",
-                                name);
-                if (-1 == ret) {
-                        gf_log (trans->xl->name, GF_LOG_ERROR,
-                                "gf_asprintf failed while setting error msg");
-                        goto fail;
-                }
-                ret = dict_set_dynstr (reply, "ERROR", msg);
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = ENOENT;
-                goto fail;
-        }
-
-        if (conf->verify_volfile_checksum) {
-                ret = dict_get_uint32 (params, "volfile-checksum", &checksum);
-                if (ret == 0) {
-                        ret = dict_get_str (params, "volfile-key",
-                                            &volfile_key);
-
-                        ret = _validate_volfile_checksum (trans->xl,
-                                                          volfile_key,
-                                                          checksum);
-                        if (-1 == ret) {
-                                ret = dict_set_str (reply, "ERROR",
-                                                    "volume-file checksum "
-                                                    "varies from earlier "
-                                                    "access");
-                                if (ret < 0)
-                                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                                "failed to set error msg");
-
-                                op_ret   = -1;
-                                op_errno = ESTALE;
-                                goto fail;
-                        }
-                }
-        }
-
-
-        peerinfo = &trans->peerinfo;
-        ret = dict_set_static_ptr (params, "peer-info", peerinfo);
-        if (ret < 0)
-                gf_log (trans->xl->name, GF_LOG_DEBUG,
-                        "failed to set peer-info");
-        ret = dict_set_str (params, "peer-info-name", peerinfo->identifier);
-        if (ret < 0)
-                gf_log (trans->xl->name, GF_LOG_DEBUG,
-                        "failed to set peer-info-name");
-
-        if (conf->auth_modules == NULL) {
-                gf_log (trans->xl->name, GF_LOG_ERROR,
-                        "Authentication module not initialized");
-        }
-
-        ret = gf_authenticate (params, config_params,
-                               conf->auth_modules);
-        if (ret == AUTH_ACCEPT) {
-                gf_log (trans->xl->name, GF_LOG_INFO,
-                        "accepted client from %s",
-                        peerinfo->identifier);
-                op_ret = 0;
-                conn->bound_xl = xl;
-                ret = dict_set_str (reply, "ERROR", "Success");
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-        } else {
-                gf_log (trans->xl->name, GF_LOG_ERROR,
-                        "Cannot authenticate client from %s",
-                        peerinfo->identifier);
-                op_ret = -1;
-                op_errno = EACCES;
-                ret = dict_set_str (reply, "ERROR", "Authentication failed");
-                if (ret < 0)
-                        gf_log (bound_xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                goto fail;
-        }
-
-        if (conn->bound_xl == NULL) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "Check volfile and handshake "
-                                    "options in protocol/client");
-                if (ret < 0)
-                        gf_log (trans->xl->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EACCES;
-                goto fail;
-        }
-
-        if ((conn->bound_xl != NULL) &&
-            (ret >= 0)                   &&
-            (conn->bound_xl->itable == NULL)) {
-                /* create inode table for this bound_xl, if one doesn't
-                   already exist */
-                lru_limit = INODE_LRU_LIMIT (frame->this);
-
-                gf_log (trans->xl->name, GF_LOG_TRACE,
-                        "creating inode table with lru_limit=%"PRId32", "
-                        "xlator=%s", lru_limit, conn->bound_xl->name);
-
-                conn->bound_xl->itable =
-                        inode_table_new (lru_limit,
-                                         conn->bound_xl);
-        }
-
-        ret = dict_set_str (reply, "process-uuid",
-                            xl->ctx->process_uuid);
-
-        ret = dict_set_uint64 (reply, "transport-ptr",
-                               ((uint64_t) (long) trans));
-
-fail:
-        dict_len = dict_serialized_length (reply);
-        if (dict_len < 0) {
-                gf_log ("server", GF_LOG_DEBUG,
-                        "failed to get serialized length of reply dict");
-                op_ret   = -1;
-                op_errno = EINVAL;
-                dict_len = 0;
-        }
-
-        rsp_hdr    = gf_hdr_new (rsp, dict_len);
-        rsp_hdrlen = gf_hdr_len (rsp, dict_len);
-        rsp = gf_param (rsp_hdr);
-
-        if (dict_len) {
-                ret = dict_serialize (reply, rsp->buf);
-                if (ret < 0) {
-                        gf_log ("server", GF_LOG_DEBUG,
-                                "failed to serialize reply dict");
-                        op_ret = -1;
-                        op_errno = -ret;
-                }
-        }
-        rsp->dict_len = hton32 (dict_len);
-
-        rsp_hdr->rsp.op_ret = hton32 (op_ret);
-        gf_errno = gf_errno_to_error (op_errno);
-        rsp_hdr->rsp.op_errno = hton32 (gf_errno);
-
-        protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_SETVOLUME,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        dict_unref (params);
-        dict_unref (reply);
-        dict_unref (config_params);
-
-        return 0;
-}
-
-
-static int
-mop_ping (call_frame_t *frame, xlator_t *bound_xl,
-          gf_hdr_common_t *hdr, size_t hdrlen,
-          struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;
-        size_t               rsp_hdrlen = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        hdr->rsp.op_ret = 0;
-
-        protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_PING,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-mop_log (call_frame_t *frame, xlator_t *bound_xl,
-         gf_hdr_common_t *hdr, size_t hdrlen,
-         struct iobuf *iobuf)
-{
-        gf_mop_log_req_t *    req = NULL;
-        char *                msg = NULL;
-        uint32_t           msglen = 0;
-
-        transport_t *       trans = NULL;
-
-        trans = TRANSPORT_FROM_FRAME (frame);
-
-        req    = gf_param (hdr);
-        msglen = ntoh32 (req->msglen);
-
-        if (msglen)
-                msg = req->msg;
-
-        gf_log_from_client (msg, trans->peerinfo.identifier);
-
-        return 0;
-}
-
-
-/* ENOSYS operations (for backword compatibility) */
-static int
-server_setdents (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_SETDENTS,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/* */
-static int
-server_getdents (call_frame_t *frame, xlator_t *bound_xl,
-                 gf_hdr_common_t *hdr, size_t hdrlen,
-                 struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_GETDENTS,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/* */
-static int
-server_lock_notify (call_frame_t *frame, xlator_t *bound_xl,
-                    gf_hdr_common_t *hdr, size_t hdrlen,
-                    struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LOCK_NOTIFY,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-/* */
-static int
-server_lock_fnotify (call_frame_t *frame, xlator_t *bound_xl,
-                     gf_hdr_common_t *hdr, size_t hdrlen,
-                     struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_FOP_REPLY, GF_PROTO_FOP_LOCK_FNOTIFY,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-mop_stats (call_frame_t *frame, xlator_t *bound_xl,
-           gf_hdr_common_t *hdr, size_t hdrlen,
-           struct iobuf *iobuf)
-{
-        gf_hdr_common_t     *rsp_hdr = NULL;
-        gf_mop_ping_rsp_t   *rsp = NULL;     /* Using for  NULL */
-        size_t               rsp_hdrlen = 0;
-        int32_t              gf_errno = 0;
-
-        rsp_hdrlen = gf_hdr_len (rsp, 0);
-        rsp_hdr    = gf_hdr_new (rsp, 0);
-
-        gf_errno = gf_errno_to_error (ENOSYS);
-        hdr->rsp.op_errno = hton32 (gf_errno);
-        hdr->rsp.op_ret = -1;
-
-        protocol_server_reply (frame, GF_OP_TYPE_MOP_REPLY, GF_MOP_STATS,
-                               rsp_hdr, rsp_hdrlen, NULL, 0, NULL);
-
-        return 0;
-}
-
-
-/*
- * get_frame_for_transport - get call frame for specified transport object
- *
- * @trans: transport object
- *
- */
-static call_frame_t *
-get_frame_for_transport (transport_t *trans)
-{
-        call_frame_t         *frame = NULL;
-        call_pool_t          *pool = NULL;
-        server_connection_t  *conn = NULL;
-        server_state_t       *state = NULL;;
-
-        GF_VALIDATE_OR_GOTO("server", trans, out);
-
-        if (trans->xl && trans->xl->ctx)
-                pool = trans->xl->ctx->pool;
-        GF_VALIDATE_OR_GOTO("server", pool, out);
-
-        frame = create_frame (trans->xl, pool);
-        GF_VALIDATE_OR_GOTO("server", frame, out);
-
-        state = GF_CALLOC (1, sizeof (*state),
-                           gf_server_mt_server_state_t);
-        GF_VALIDATE_OR_GOTO("server", state, out);
-
-        conn = trans->xl_private;
-        if (conn) {
-                if (conn->bound_xl)
-                        state->itable = conn->bound_xl->itable;
-                state->bound_xl = conn->bound_xl;
-        }
-
-        state->trans = transport_ref (trans);
-        state->resolve.fd_no = -1;
-        state->resolve2.fd_no = -1;
-
-        frame->root->trans = conn;
-        frame->root->state = state;        /* which socket */
-        frame->root->unique = 0;           /* which call */
-
-out:
-        return frame;
-}
-
-
-static int
-server_decode_groups (call_frame_t *frame, gf_hdr_common_t *hdr)
-{
-        int     i = 0;
-
-        if ((!frame) || (!hdr))
-                return 0;
-
-        frame->root->ngrps = ntoh32 (hdr->req.ngrps);
-        if (frame->root->ngrps == 0)
-                return 0;
-
-        if (frame->root->ngrps > GF_REQUEST_MAXGROUPS)
-                return -1;
-
-        for (; i < frame->root->ngrps; ++i)
-                frame->root->groups[i] = ntoh32 (hdr->req.groups[i]);
-
-        return 0;
-}
-
-
-/*
- * get_frame_for_call - create a frame into the capable of
- *                      generating and replying the reply packet by itself.
- *                      By making a call with this frame, the last UNWIND
- *                      function will have all needed state from its
- *                      frame_t->root to send reply.
- * @trans:
- * @blk:
- * @params:
- *
- * not for external reference
- */
-static call_frame_t *
-get_frame_for_call (transport_t *trans, gf_hdr_common_t *hdr)
-{
-        call_frame_t *frame = NULL;
-        int32_t      ret = -1;
-
-        frame = get_frame_for_transport (trans);
-
-        frame->root->op   = ntoh32 (hdr->op);
-        frame->root->type = ntoh32 (hdr->type);
-
-        frame->root->uid         = ntoh32 (hdr->req.uid);
-        frame->root->unique      = ntoh64 (hdr->callid);      /* which call */
-        frame->root->gid         = ntoh32 (hdr->req.gid);
-        frame->root->pid         = ntoh32 (hdr->req.pid);
-        frame->root->lk_owner    = ntoh64 (hdr->req.lk_owner);
-        ret = server_decode_groups (frame, hdr);
-
-        if (ret) {
-                //FRAME_DESTROY (frame);
-                return NULL;
-        }
-
-        return frame;
-}
-
-/*
- * prototype of operations function for each of mop and
- * fop at server protocol level
- *
- * @frame: call frame pointer
- * @bound_xl: the xlator that this frame is bound to
- * @params: parameters dictionary
- *
- * to be used by protocol interpret, _not_ for exterenal reference
- */
-typedef int32_t (*gf_op_t) (call_frame_t *frame, xlator_t *bould_xl,
-                            gf_hdr_common_t *hdr, size_t hdrlen,
-                            struct iobuf *iobuf);
-
-
-static gf_op_t gf_fops[] = {
-        [GF_PROTO_FOP_STAT]         =  server_stat,
-        [GF_PROTO_FOP_READLINK]     =  server_readlink,
-        [GF_PROTO_FOP_MKNOD]        =  server_mknod,
-        [GF_PROTO_FOP_MKDIR]        =  server_mkdir,
-        [GF_PROTO_FOP_UNLINK]       =  server_unlink,
-        [GF_PROTO_FOP_RMDIR]        =  server_rmdir,
-        [GF_PROTO_FOP_SYMLINK]      =  server_symlink,
-        [GF_PROTO_FOP_RENAME]       =  server_rename,
-        [GF_PROTO_FOP_LINK]         =  server_link,
-        [GF_PROTO_FOP_TRUNCATE]     =  server_truncate,
-        [GF_PROTO_FOP_OPEN]         =  server_open,
-        [GF_PROTO_FOP_READ]         =  server_readv,
-        [GF_PROTO_FOP_WRITE]        =  server_writev,
-        [GF_PROTO_FOP_STATFS]       =  server_statfs,
-        [GF_PROTO_FOP_FLUSH]        =  server_flush,
-        [GF_PROTO_FOP_FSYNC]        =  server_fsync,
-        [GF_PROTO_FOP_SETXATTR]     =  server_setxattr,
-        [GF_PROTO_FOP_GETXATTR]     =  server_getxattr,
-        [GF_PROTO_FOP_FGETXATTR]    =  server_fgetxattr,
-        [GF_PROTO_FOP_FSETXATTR]    =  server_fsetxattr,
-        [GF_PROTO_FOP_REMOVEXATTR]  =  server_removexattr,
-        [GF_PROTO_FOP_OPENDIR]      =  server_opendir,
-        [GF_PROTO_FOP_FSYNCDIR]     =  server_fsyncdir,
-        [GF_PROTO_FOP_ACCESS]       =  server_access,
-        [GF_PROTO_FOP_CREATE]       =  server_create,
-        [GF_PROTO_FOP_FTRUNCATE]    =  server_ftruncate,
-        [GF_PROTO_FOP_FSTAT]        =  server_fstat,
-        [GF_PROTO_FOP_LK]           =  server_lk,
-        [GF_PROTO_FOP_LOOKUP]       =  server_lookup,
-        [GF_PROTO_FOP_READDIR]      =  server_readdir,
-        [GF_PROTO_FOP_READDIRP]     =  server_readdirp,
-        [GF_PROTO_FOP_INODELK]      =  server_inodelk,
-        [GF_PROTO_FOP_FINODELK]     =  server_finodelk,
-        [GF_PROTO_FOP_ENTRYLK]      =  server_entrylk,
-        [GF_PROTO_FOP_FENTRYLK]     =  server_fentrylk,
-        [GF_PROTO_FOP_CHECKSUM]     =  server_checksum,
-        [GF_PROTO_FOP_RCHECKSUM]    =  server_rchecksum,
-        [GF_PROTO_FOP_XATTROP]      =  server_xattrop,
-        [GF_PROTO_FOP_FXATTROP]     =  server_fxattrop,
-        [GF_PROTO_FOP_SETATTR]      =  server_setattr,
-        [GF_PROTO_FOP_FSETATTR]     =  server_fsetattr,
-        [GF_PROTO_FOP_SETDENTS]     =  server_setdents,
-        [GF_PROTO_FOP_GETDENTS]     =  server_getdents,
-        [GF_PROTO_FOP_LOCK_NOTIFY]  =  server_lock_notify,
-        [GF_PROTO_FOP_LOCK_FNOTIFY] =  server_lock_fnotify,
-};
-
-
-
-static gf_op_t gf_mops[] = {
-        [GF_MOP_SETVOLUME] = mop_setvolume,
-        [GF_MOP_GETVOLUME] = mop_getvolume,
-        [GF_MOP_GETSPEC]   = mop_getspec,
-        [GF_MOP_PING]      = mop_ping,
-        [GF_MOP_LOG]       = mop_log,
-        [GF_MOP_STATS]     = mop_stats,
-};
-
-static gf_op_t gf_cbks[] = {
-        [GF_CBK_FORGET]     = server_forget,
-        [GF_CBK_RELEASE]    = server_release,
-        [GF_CBK_RELEASEDIR] = server_releasedir
-};
-
-static int
-protocol_server_interpret (xlator_t *this, transport_t *trans,
-                           char *hdr_p, size_t hdrlen, struct iobuf *iobuf)
-{
-        server_connection_t         *conn = NULL;
-        gf_hdr_common_t             *hdr = NULL;
-        xlator_t                    *bound_xl = NULL;
-        call_frame_t                *frame = NULL;
-        peer_info_t                 *peerinfo = NULL;
-        int32_t                      type = -1;
-        int32_t                      op = -1;
-        int32_t                      ret = -1;
-
-        hdr  = (gf_hdr_common_t *)hdr_p;
-        type = ntoh32 (hdr->type);
-        op   = ntoh32 (hdr->op);
-
-        conn = trans->xl_private;
-        if (conn)
-                bound_xl = conn->bound_xl;
-
-        peerinfo = &trans->peerinfo;
-        switch (type) {
-        case GF_OP_TYPE_FOP_REQUEST:
-                if ((op < 0) || (op >= GF_PROTO_FOP_MAXVALUE)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid fop %"PRId32" from client %s",
-                                op, peerinfo->identifier);
-                        break;
-                }
-                if (bound_xl == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Received fop %"PRId32" before "
-                                "authentication.", op);
-                        break;
-                }
-                frame = get_frame_for_call (trans, hdr);
-                if (!frame) {
-                        ret = -1;
-                        goto out;
-                }
-		frame->op = op;
-                ret = gf_fops[op] (frame, bound_xl, hdr, hdrlen, iobuf);
-                break;
-
-        case GF_OP_TYPE_MOP_REQUEST:
-                if ((op < 0) || (op >= GF_MOP_MAXVALUE)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid mop %"PRId32" from client %s",
-                                op, peerinfo->identifier);
-                        break;
-                }
-                frame = get_frame_for_call (trans, hdr);
-                if (!frame) {
-                        ret = -1;
-                        goto out;
-                }
-		frame->op = op;
-                ret = gf_mops[op] (frame, bound_xl, hdr, hdrlen, iobuf);
-                break;
-
-        case GF_OP_TYPE_CBK_REQUEST:
-                if ((op < 0) || (op >= GF_CBK_MAXVALUE)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid cbk %"PRId32" from client %s",
-                                op, peerinfo->identifier);
-                        break;
-                }
-                if (bound_xl == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Received cbk %d before authentication.", op);
-                        break;
-                }
-
-                frame = get_frame_for_call (trans, hdr);
-                if (!frame) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = gf_cbks[op] (frame, bound_xl, hdr, hdrlen, iobuf);
-                break;
-
-        default:
-                break;
-        }
-out:
-        return ret;
-}
-
-
-/*
- * server_fd - fdtable dump function for server protocol
- * @this:
- *
- */
-static int
-server_fd (xlator_t *this)
-{
-         server_conf_t        *conf = NULL;
-         server_connection_t  *trav = NULL;
-         char                 key[GF_DUMP_MAX_BUF_LEN];
-         int                  i = 1;
-         int                  ret = -1;
-
-         if (!this)
-                 return -1;
-
-         conf = this->private;
-         if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-         }
-
-         gf_proc_dump_add_section("xlator.protocol.server.conn");
-
-         ret = pthread_mutex_trylock (&conf->mutex);
-         if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump fdtable"
-                " errno: %d", errno);
-                return -1;
-        }
-
-         list_for_each_entry (trav, &conf->conns, list) {
-                 if (trav->id) {
-                         gf_proc_dump_build_key(key,
-                                          "xlator.protocol.server.conn",
-                                          "%d.id", i);
-                         gf_proc_dump_write(key, "%s", trav->id);
-                 }
-
-                 gf_proc_dump_build_key(key,"xlator.protocol.server.conn",
-                                        "%d.ref",i)
-                 gf_proc_dump_write(key, "%d", trav->ref);
-                 if (trav->bound_xl) {
-                         gf_proc_dump_build_key(key,
-                                          "xlator.protocol.server.conn",
-                                          "%d.bound_xl", i);
-                         gf_proc_dump_write(key, "%s", trav->bound_xl->name);
-                 }
-
-                 gf_proc_dump_build_key(key,
-                                        "xlator.protocol.server.conn",
-                                         "%d.id", i);
-                 fdtable_dump(trav->fdtable,key);
-                 i++;
-         }
-        pthread_mutex_unlock (&conf->mutex);
-
-
-        return 0;
- }
-
-static int
-server_priv (xlator_t *this)
-{
-        return 0;
-}
-
-static int
-server_inode (xlator_t *this)
-{
-         server_conf_t        *conf = NULL;
-         server_connection_t  *trav = NULL;
-         char                 key[GF_DUMP_MAX_BUF_LEN];
-         int                  i = 1;
-         int                  ret = -1;
-
-         if (!this)
-                 return -1;
-
-         conf = this->private;
-         if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-         }
-
-         ret = pthread_mutex_trylock (&conf->mutex);
-         if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump itable"
-                " errno: %d", errno);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                 if (trav->bound_xl && trav->bound_xl->itable) {
-                         gf_proc_dump_build_key(key,
-                                          "xlator.protocol.server.conn",
-                                          "%d.bound_xl.%s",
-                                          i, trav->bound_xl->name);
-                         inode_table_dump(trav->bound_xl->itable,key);
-                         i++;
-                 }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-
-        return 0;
-}
-
-
-static void
-get_auth_types (dict_t *this, char *key, data_t *value, void *data)
-{
-        dict_t   *auth_dict = NULL;
-        char     *saveptr = NULL;
-        char     *tmp = NULL;
-        char     *key_cpy = NULL;
-        int32_t   ret = -1;
-
-        auth_dict = data;
-        key_cpy = gf_strdup (key);
-        GF_VALIDATE_OR_GOTO("server", key_cpy, out);
-
-        tmp = strtok_r (key_cpy, ".", &saveptr);
-        ret = strcmp (tmp, "auth");
-        if (ret == 0) {
-                tmp = strtok_r (NULL, ".", &saveptr);
-                if (strcmp (tmp, "ip") == 0) {
-                        /* TODO: backward compatibility, remove when
-                           newer versions are available */
-                        tmp = "addr";
-                        gf_log ("server", GF_LOG_WARNING,
-                                "assuming 'auth.ip' to be 'auth.addr'");
-                }
-                ret = dict_set_dynptr (auth_dict, tmp, NULL, 0);
-                if (ret < 0) {
-                        gf_log ("server", GF_LOG_DEBUG,
-                                "failed to dict_set_dynptr");
-                }
-        }
-
-        GF_FREE (key_cpy);
-out:
-        return;
-}
-
-
-static int
-validate_auth_options (xlator_t *this, dict_t *dict)
-{
-        int            ret = -1;
-        int            error = 0;
-        xlator_list_t *trav = NULL;
-        data_pair_t   *pair = NULL;
-        char          *saveptr = NULL;
-        char          *tmp = NULL;
-        char          *key_cpy = NULL;
-
-        trav = this->children;
-        while (trav) {
-                error = -1;
-                for (pair = dict->members_list; pair; pair = pair->next) {
-                        key_cpy = gf_strdup (pair->key);
-                        tmp = strtok_r (key_cpy, ".", &saveptr);
-                        ret = strcmp (tmp, "auth");
-                        if (ret == 0) {
-                                /* for module type */
-                                tmp = strtok_r (NULL, ".", &saveptr);
-                                /* for volume name */
-                                tmp = strtok_r (NULL, ".", &saveptr);
-                        }
-
-                        if (strcmp (tmp, trav->xlator->name) == 0) {
-                                error = 0;
-                                GF_FREE (key_cpy);
-                                break;
-                        }
-                        GF_FREE (key_cpy);
-                }
-                if (-1 == error) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "volume '%s' defined as subvolume, but no "
-                                "authentication defined for the same",
-                                trav->xlator->name);
-                        break;
-                }
-                trav = trav->next;
-        }
-
-        return error;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_server_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-
-/*
- * init - called during server protocol initialization
- *
- * @this:
- *
- */
-int
-init (xlator_t *this)
-{
-        int32_t        ret = -1;
-        transport_t   *trans = NULL;
-        server_conf_t *conf = NULL;
-        data_t        *data = NULL;
-        data_t        *trace = NULL;
-
-        if (this->children == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "protocol/server should have subvolume");
-                goto out;
-        }
-
-        trans = transport_load (this->options, this);
-        if (trans == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to load transport");
-                goto out;
-        }
-
-        ret = transport_listen (trans);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to bind/listen on socket");
-                goto out;
-        }
-
-        conf = GF_CALLOC (1, sizeof (server_conf_t),
-                          gf_server_mt_server_conf_t);
-        GF_VALIDATE_OR_GOTO(this->name, conf, out);
-
-        INIT_LIST_HEAD (&conf->conns);
-        pthread_mutex_init (&conf->mutex, NULL);
-
-        conf->trans = trans;
-
-        conf->auth_modules = dict_new ();
-        GF_VALIDATE_OR_GOTO(this->name, conf->auth_modules, out);
-
-        dict_foreach (this->options, get_auth_types,
-                      conf->auth_modules);
-        ret = validate_auth_options (this, this->options);
-        if (ret == -1) {
-                /* logging already done in validate_auth_options function. */
-                goto out;
-        }
-
-        ret = gf_auth_init (this, conf->auth_modules);
-        if (ret) {
-                dict_unref (conf->auth_modules);
-                goto out;
-        }
-
-        this->private = conf;
-
-        ret = dict_get_int32 (this->options, "inode-lru-limit",
-                              &conf->inode_lru_limit);
-        if (ret < 0) {
-                conf->inode_lru_limit = 1024;
-        }
-
-        ret = dict_get_int32 (this->options, "limits.transaction-size",
-                              &conf->max_block_size);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "defaulting limits.transaction-size to %d",
-                        DEFAULT_BLOCK_SIZE);
-                conf->max_block_size = DEFAULT_BLOCK_SIZE;
-        }
-
-        conf->verify_volfile_checksum = 1;
-        data = dict_get (this->options, "verify-volfile-checksum");
-        if (data) {
-                ret = gf_string2boolean(data->data,
-                                        &conf->verify_volfile_checksum);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "wrong value for verify-volfile-checksum");
-                        conf->verify_volfile_checksum = 1;
-                }
-        }
-
-        trace = dict_get (this->options, "trace");
-	if (trace) {
-		if (gf_string2boolean (trace->data,
-				       &conf->trace) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"'trace' takes on only boolean values.");
-			return -1;
-		}
-	}
-
-#ifndef GF_DARWIN_HOST_OS
-        {
-                struct rlimit lim;
-
-                lim.rlim_cur = 1048576;
-                lim.rlim_max = 1048576;
-
-                if (setrlimit (RLIMIT_NOFILE, &lim) == -1) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "WARNING: Failed to set 'ulimit -n 1M': %s",
-                                strerror(errno));
-                        lim.rlim_cur = 65536;
-                        lim.rlim_max = 65536;
-
-                        if (setrlimit (RLIMIT_NOFILE, &lim) == -1) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Failed to set max open fd to 64k: %s",
-                                        strerror(errno));
-                        } else {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "max open fd set to 64k");
-                        }
-                }
-        }
-#endif
-        this->graph->top = this;
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-
-static int
-protocol_server_pollin (xlator_t *this, transport_t *trans)
-{
-        char                *hdr = NULL;
-        size_t               hdrlen = 0;
-        int                  ret = -1;
-        struct iobuf        *iobuf = NULL;
-
-
-        ret = transport_receive (trans, &hdr, &hdrlen, &iobuf);
-
-        if (ret == 0)
-                ret = protocol_server_interpret (this, trans, hdr,
-                                                 hdrlen, iobuf);
-
-        /* TODO: use mem-pool */
-        GF_FREE (hdr);
-
-        return ret;
-}
-
-
-/*
- * fini - finish function for server protocol, called before
- *        unloading server protocol.
- *
- * @this:
- *
- */
-void
-fini (xlator_t *this)
-{
-        server_conf_t *conf = this->private;
-
-        GF_VALIDATE_OR_GOTO(this->name, conf, out);
-
-        if (conf->auth_modules) {
-                dict_unref (conf->auth_modules);
-        }
-
-        GF_FREE (conf);
-        this->private = NULL;
-out:
-        return;
-}
-
-/*
- * server_protocol_notify - notify function for server protocol
- * @this:
- * @trans:
- * @event:
- *
- */
-int
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int          ret = 0;
-        transport_t *trans = NULL;
-        peer_info_t *peerinfo = NULL;
-        peer_info_t *myinfo = NULL;
-
-        THIS = this;
-        trans = data;
-        if (!trans) {
-                gf_log (this->name, GF_LOG_ERROR, "!trans");
-                goto out;
-        }
-
-        peerinfo = &(trans->peerinfo);
-        myinfo = &(trans->myinfo);
-
-        switch (event) {
-        case GF_EVENT_POLLIN:
-                ret = protocol_server_pollin (this, trans);
-                break;
-        case GF_EVENT_POLLERR:
-        {
-                gf_log (trans->xl->name, GF_LOG_INFO, "%s disconnected",
-                        peerinfo->identifier);
-
-                ret = -1;
-                transport_disconnect (trans);
-                if (trans->xl_private == NULL) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "POLLERR received on (%s) even before "
-                                "handshake with (%s) is successful",
-                                myinfo->identifier, peerinfo->identifier);
-                } else {
-                        /*
-                         * FIXME: shouldn't we check for return value?
-                         * what should be done if cleanup fails?
-                         */
-                        gf_server_connection_cleanup (this, trans->xl_private);
-                }
-        }
-        break;
-
-        case GF_EVENT_TRANSPORT_CLEANUP:
-        {
-                if (trans->xl_private) {
-                        gf_server_connection_put (this, trans->xl_private);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "transport (%s) cleaned up even before "
-                                "handshake with (%s) is successful",
-                                myinfo->identifier, peerinfo->identifier);
-                }
-        }
-        break;
-
-        default:
-                default_notify (this, event, data);
-                break;
-        }
-out:
-        return ret;
-}
-
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct xlator_dumpops dumpops = {
-        .inode = server_inode,
-        .priv  = server_priv,
-        .fd    = server_fd,
-};
-
-
-struct volume_options options[] = {
-        { .key   = {"transport-type"},
-          .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp",
-                    "tcp/server", "ib-verbs/server"},
-          .type  = GF_OPTION_TYPE_STR
-        },
-        { .key   = {"volume-filename.*"},
-          .type  = GF_OPTION_TYPE_PATH,
-        },
-        { .key   = {"inode-lru-limit"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 0,
-          .max   = (1 * GF_UNIT_MB)
-        },
-        { .key   = {"client-volume-filename"},
-          .type  = GF_OPTION_TYPE_PATH
-        },
-        { .key   = {"verify-volfile-checksum"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"trace"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"conf-dir"},
-          .type  = GF_OPTION_TYPE_PATH,
-        },
-
-        { .key   = {NULL} },
-};
diff --git a/xlators/protocol/legacy/server/src/server-protocol.h b/xlators/protocol/legacy/server/src/server-protocol.h
deleted file mode 100644
index be6776033..000000000
--- a/xlators/protocol/legacy/server/src/server-protocol.h
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SERVER_PROTOCOL_H_
-#define _SERVER_PROTOCOL_H_
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <pthread.h>
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
-#include "server-mem-types.h"
-#include "authenticate.h"
-#include "transport.h"
-
-#define DEFAULT_BLOCK_SIZE         4194304   /* 4MB */
-#define DEFAULT_VOLUME_FILE_PATH   CONFDIR "/glusterfs.vol"
-
-typedef struct _server_state server_state_t;
-
-struct _locker {
-	struct list_head  lockers;
-        char             *volume;
-	loc_t             loc;
-	fd_t             *fd;
-	pid_t             pid;
-};
-
-struct _lock_table {
-	struct list_head  file_lockers;
-	struct list_head  dir_lockers;
-	gf_lock_t         lock;
-	size_t            count;
-};
-
-
-/* private structure per connection (transport object)
- * used as transport_t->xl_private
- */
-struct _server_connection {
-	struct list_head    list;
-	char               *id;
-	int                 ref;
-        int                 active_transports;
-	pthread_mutex_t     lock;
-	char                disconnected;
-	fdtable_t          *fdtable; 
-	struct _lock_table *ltable;
-	xlator_t           *bound_xl;
-};
-
-typedef struct _server_connection server_connection_t;
-
-
-server_connection_t *
-gf_server_connection_get (xlator_t *this, const char *id);
-
-void
-gf_server_connection_put (xlator_t *this, server_connection_t *conn);
-
-int
-gf_server_connection_cleanup (xlator_t *this, server_connection_t *conn);
-
-struct _volfile_ctx {
-        struct _volfile_ctx *next;
-        char                *key;
-        uint32_t             checksum;
-};
-
-typedef struct {
-        struct _volfile_ctx *volfile;
-
-	dict_t           *auth_modules;
-	transport_t      *trans;
-	int32_t           max_block_size;
-	int32_t           inode_lru_limit;
-	pthread_mutex_t   mutex;
-	struct list_head  conns;
-        gf_boolean_t      verify_volfile_checksum;
-        gf_boolean_t      trace;
-} server_conf_t;
-
-
-typedef enum {
-        RESOLVE_MUST = 1,
-        RESOLVE_NOT,
-        RESOLVE_MAY,
-        RESOLVE_DONTCARE,
-        RESOLVE_EXACT
-} server_resolve_type_t;
-
-
-struct resolve_comp {
-        char      *basename;
-        ino_t      ino;
-        uint64_t   gen;
-        inode_t   *inode;
-};
-
-typedef struct {
-        server_resolve_type_t  type;
-        uint64_t               fd_no;
-        ino_t                  ino;
-        uint64_t               gen;
-        ino_t                  par;
-        char                  *path;
-        char                  *bname;
-	char                  *resolved;
-        int                    op_ret;
-        int                    op_errno;
-        loc_t                  deep_loc;
-        struct resolve_comp   *components;
-        int                    comp_count;
-} server_resolve_t;
-
-
-typedef int (*server_resume_fn_t) (call_frame_t *frame, xlator_t *bound_xl);
-
-int
-gf_resolve_and_resume (call_frame_t *frame, server_resume_fn_t fn);
-
-struct _server_state {
-	transport_t      *trans;
-	xlator_t         *bound_xl;
-        inode_table_t    *itable;
-
-        server_resume_fn_t resume_fn;
-
-	loc_t             loc;
-	loc_t             loc2;
-        server_resolve_t  resolve;
-        server_resolve_t  resolve2;
-
-        /* used within resolve_and_resume */
-        loc_t            *loc_now;
-        server_resolve_t *resolve_now;
-
-        struct iatt       stbuf;
-        int               valid;
-
-	fd_t             *fd;
-        dict_t           *params;
-	int               flags;
-        int               wbflags;
-        struct iobuf     *iobuf;
-        struct iobref    *iobref;
-
-	size_t            size;
-	off_t             offset;
-	mode_t            mode;
-	dev_t             dev;
-	size_t            nr_count;
-	int               cmd;
-	int               type;
-	char             *name;
-	int               name_len;
-
-	int               mask;
-	char              is_revalidate;
-	dict_t           *dict;
-	struct gf_flock      flock;
-        const char       *volume;
-        dir_entry_t      *entry;
-};
-
-
-#endif
diff --git a/xlators/protocol/legacy/server/src/server-resolve.c b/xlators/protocol/legacy/server/src/server-resolve.c
deleted file mode 100644
index 5a02be7ef..000000000
--- a/xlators/protocol/legacy/server/src/server-resolve.c
+++ /dev/null
@@ -1,658 +0,0 @@
-/*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "server-protocol.h"
-#include "server-helpers.h"
-
-#include "compat-errno.h"
-
-void
-gf_server_print_request (call_frame_t *frame);
-
-static int
-server_resolve_all (call_frame_t *frame);
-static int
-resolve_entry_simple (call_frame_t *frame);
-static int
-resolve_inode_simple (call_frame_t *frame);
-static int
-resolve_path_simple (call_frame_t *frame);
-
-
-static int
-component_count (const char *path)
-{
-        int         count = 0;
-        const char *trav = NULL;
-
-        trav = path;
-
-        for (trav = path; *trav; trav++) {
-                if (*trav == '/')
-                        count++;
-        }
-
-        return count + 2;
-}
-
-
-static int
-prepare_components (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        xlator_t             *this = NULL;
-        server_resolve_t     *resolve = NULL;
-        char                 *resolved = NULL;
-        int                   count = 0;
-        struct resolve_comp  *components = NULL;
-        int                   i = 0;
-        char                 *trav = NULL;
-
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        resolved = gf_strdup (resolve->path);
-        resolve->resolved = resolved;
-
-        count = component_count (resolve->path);
-        components = GF_CALLOC (sizeof (*components), count,
-                                gf_server_mt_resolve_comp);
-        resolve->components = components;
-
-        components[0].basename = "";
-        components[0].ino      = 1;
-        components[0].gen      = 0;
-        components[0].inode    = state->itable->root;
-
-        i = 1;
-        for (trav = resolved; *trav; trav++) {
-                if (*trav == '/') {
-                        components[i].basename = trav + 1;
-                        *trav = 0;
-                        i++;
-                }
-        }
-
-        return 0;
-}
-
-
-static int
-resolve_loc_touchup (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        server_resolve_t     *resolve = NULL;
-        loc_t                *loc = NULL;
-        char                 *path = NULL;
-        int                   ret = 0;
-
-        state = CALL_STATE (frame);
-
-        resolve = state->resolve_now;
-        loc     = state->loc_now;
-
-        if (!loc->path) {
-                if (loc->parent && resolve->bname) {
-                        ret = inode_path (loc->parent, resolve->bname, &path);
-                } else if (loc->inode) {
-                        ret = inode_path (loc->inode, NULL, &path);
-                }
-
-                if (!path)
-                        path = gf_strdup (resolve->path);
-
-                loc->path = path;
-        }
-
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-
-        if (!loc->parent && loc->inode) {
-                loc->parent = inode_parent (loc->inode, 0, NULL);
-        }
-
-        return 0;
-}
-
-
-static int
-resolve_deep_continue (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        xlator_t             *this = NULL;
-        server_resolve_t     *resolve = NULL;
-        int                   ret = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        resolve->op_ret   = 0;
-        resolve->op_errno = 0;
-
-        if (resolve->par)
-                ret = resolve_entry_simple (frame);
-        else if (resolve->ino)
-                ret = resolve_inode_simple (frame);
-        else if (resolve->path)
-                ret = resolve_path_simple (frame);
-
-        resolve_loc_touchup (frame);
-
-        server_resolve_all (frame);
-
-        return 0;
-}
-
-
-static int
-resolve_deep_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-                  dict_t *xattr, struct iatt *postparent)
-{
-        server_state_t       *state = NULL;
-        server_resolve_t     *resolve = NULL;
-        struct resolve_comp  *components = NULL;
-        int                   i = 0;
-        inode_t              *link_inode = NULL;
-
-        state = CALL_STATE (frame);
-        resolve = state->resolve_now;
-        components = resolve->components;
-
-        i = (long) cookie;
-
-        if (op_ret == -1) {
-                goto get_out_of_here;
-        }
-
-        if (i != 0) {
-                /* no linking for root inode */
-                link_inode = inode_link (inode, resolve->deep_loc.parent,
-                                         resolve->deep_loc.name, buf);
-                inode_lookup (link_inode);
-                components[i].inode  = link_inode;
-                link_inode = NULL;
-        }
-
-        loc_wipe (&resolve->deep_loc);
-
-        i++; /* next component */
-
-        if (!components[i].basename) {
-                /* all components of the path are resolved */
-                goto get_out_of_here;
-        }
-
-        /* join the current component with the path resolved until now */
-        *(components[i].basename - 1) = '/';
-
-        resolve->deep_loc.path   = gf_strdup (resolve->resolved);
-        resolve->deep_loc.parent = inode_ref (components[i-1].inode);
-        resolve->deep_loc.inode  = inode_new (state->itable);
-        resolve->deep_loc.name   = components[i].basename;
-
-        STACK_WIND_COOKIE (frame, resolve_deep_cbk, (void *) (long) i,
-                           BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
-                           &resolve->deep_loc, NULL);
-        return 0;
-
-get_out_of_here:
-        resolve_deep_continue (frame);
-        return 0;
-}
-
-
-static int
-resolve_path_deep (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-        int                 i = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        gf_log (BOUND_XL (frame)->name, GF_LOG_DEBUG,
-                "RESOLVE %s() seeking deep resolution of %s",
-                gf_fop_list[frame->root->op], resolve->path);
-
-        prepare_components (frame);
-
-        /* start from the root */
-        resolve->deep_loc.inode = state->itable->root;
-        resolve->deep_loc.path  = gf_strdup ("/");
-        resolve->deep_loc.name  = "";
-
-        STACK_WIND_COOKIE (frame, resolve_deep_cbk, (void *) (long) i,
-                           BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
-                           &resolve->deep_loc, NULL);
-        return 0;
-}
-
-
-static int
-resolve_path_simple (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        xlator_t             *this = NULL;
-        server_resolve_t     *resolve = NULL;
-        struct resolve_comp  *components = NULL;
-        int                   ret = -1;
-        int                   par_idx = 0;
-        int                   ino_idx = 0;
-        int                   i = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-        components = resolve->components;
-
-        if (!components) {
-                resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
-                goto out;
-        }
-
-        for (i = 0; components[i].basename; i++) {
-                par_idx = ino_idx;
-                ino_idx = i;
-        }
-
-        if (!components[par_idx].inode) {
-                resolve->op_ret    = -1;
-                resolve->op_errno  = ENOENT;
-                goto out;
-        }
-
-        if (!components[ino_idx].inode &&
-            (resolve->type == RESOLVE_MUST || resolve->type == RESOLVE_EXACT)) {
-                resolve->op_ret    = -1;
-                resolve->op_errno  = ENOENT;
-                goto out;
-        }
-
-        if (components[ino_idx].inode && resolve->type == RESOLVE_NOT) {
-                resolve->op_ret    = -1;
-                resolve->op_errno  = EEXIST;
-                goto out;
-        }
-
-        if (components[ino_idx].inode)
-                state->loc_now->inode  = inode_ref (components[ino_idx].inode);
-        state->loc_now->parent = inode_ref (components[par_idx].inode);
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-/*
-  Check if the requirements are fulfilled by entries in the inode cache itself
-  Return value:
-  <= 0 - simple resolution was decisive and complete (either success or failure)
-  > 0  - indecisive, need to perform deep resolution
-*/
-
-static int
-resolve_entry_simple (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-        inode_t            *parent = NULL;
-        inode_t            *inode = NULL;
-        int                 ret = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        parent = inode_get (state->itable, resolve->par, 0);
-        if (!parent) {
-                /* simple resolution is indecisive. need to perform
-                   deep resolution */
-                resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
-                ret = 1;
-
-                inode = inode_grep (state->itable, parent, resolve->bname);
-                if (inode != NULL) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%"PRId64": inode "
-                                "(pointer:%p ino: %"PRIu64") present but parent"
-                                " is NULL for path (%s)", frame->root->unique,
-                                inode, inode->ino, resolve->path);
-                        inode_unref (inode);
-                }
-                goto out;
-        }
-
-//        if (parent->ino != 1 && parent->generation != resolve->gen) {
-        if (0) {
-                /* simple resolution is decisive - request was for a
-                   stale handle */
-                resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
-                ret = -1;
-                goto out;
-        }
-
-        /* expected @parent was found from the inode cache */
-        state->loc_now->parent = inode_ref (parent);
-
-        inode = inode_grep (state->itable, parent, resolve->bname);
-        if (!inode) {
-                switch (resolve->type) {
-                case RESOLVE_DONTCARE:
-                case RESOLVE_NOT:
-                        ret = 0;
-                        break;
-                case RESOLVE_MAY:
-                        ret = 1;
-                        break;
-                default:
-                        resolve->op_ret   = -1;
-                        resolve->op_errno = ENOENT;
-                        ret = 1;
-                        break;
-                }
-
-                goto out;
-        }
-
-        if (resolve->type == RESOLVE_NOT) {
-                gf_log (this->name, GF_LOG_DEBUG, "inode (pointer: %p ino:%"
-                        PRIu64") found for path (%s) while type is RESOLVE_NOT",
-                        inode, inode->ino, resolve->path);
-                resolve->op_ret   = -1;
-                resolve->op_errno = EEXIST;
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-
-        state->loc_now->inode  = inode_ref (inode);
-
-out:
-        if (parent)
-                inode_unref (parent);
-
-        if (inode)
-                inode_unref (inode);
-
-        return ret;
-}
-
-
-static int
-server_resolve_entry (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-        int                 ret = 0;
-        loc_t              *loc = NULL;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-        loc  = state->loc_now;
-
-        ret = resolve_entry_simple (frame);
-
-        if (ret > 0) {
-                loc_wipe (loc);
-                resolve_path_deep (frame);
-                return 0;
-        }
-
-        if (ret == 0)
-                resolve_loc_touchup (frame);
-
-        server_resolve_all (frame);
-
-        return 0;
-}
-
-
-static int
-resolve_inode_simple (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-        inode_t            *inode = NULL;
-        int                 ret = 0;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        if (resolve->type == RESOLVE_EXACT) {
-                inode = inode_get (state->itable, resolve->ino, resolve->gen);
-        } else {
-                inode = inode_get (state->itable, resolve->ino, 0);
-        }
-
-        if (!inode) {
-                resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
-                ret = 1;
-                goto out;
-        }
-
-//        if (inode->ino != 1 && inode->generation != resolve->gen) {
-        if (0) {
-                resolve->op_ret      = -1;
-                resolve->op_errno    = ENOENT;
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-
-        state->loc_now->inode = inode_ref (inode);
-
-out:
-        if (inode)
-                inode_unref (inode);
-
-        return ret;
-}
-
-
-static int
-server_resolve_inode (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        int                 ret = 0;
-        loc_t              *loc = NULL;
-
-        state = CALL_STATE (frame);
-        loc  = state->loc_now;
-
-        ret = resolve_inode_simple (frame);
-
-        if (ret > 0) {
-                loc_wipe (loc);
-                resolve_path_deep (frame);
-                return 0;
-        }
-
-        if (ret == 0)
-                resolve_loc_touchup (frame);
-
-        server_resolve_all (frame);
-
-        return 0;
-}
-
-
-static int
-server_resolve_fd (call_frame_t *frame)
-{
-        server_state_t       *state = NULL;
-        xlator_t             *this = NULL;
-        server_resolve_t     *resolve = NULL;
-        server_connection_t  *conn = NULL;
-        uint64_t              fd_no = -1;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-        conn  = SERVER_CONNECTION (frame);
-
-        fd_no = resolve->fd_no;
-
-        state->fd = gf_fd_fdptr_get (conn->fdtable, fd_no);
-
-        if (!state->fd) {
-                resolve->op_ret   = -1;
-                resolve->op_errno = EBADFD;
-        }
-
-        server_resolve_all (frame);
-
-        return 0;
-}
-
-
-static int
-server_resolve (call_frame_t *frame)
-{
-        server_state_t     *state = NULL;
-        xlator_t           *this = NULL;
-        server_resolve_t   *resolve = NULL;
-
-        state = CALL_STATE (frame);
-        this  = frame->this;
-        resolve = state->resolve_now;
-
-        if (resolve->fd_no != -1) {
-
-                server_resolve_fd (frame);
-
-        } else if (resolve->par) {
-
-                server_resolve_entry (frame);
-
-        } else if (resolve->ino) {
-
-                server_resolve_inode (frame);
-
-        } else if (resolve->path) {
-
-                resolve_path_deep (frame);
-
-        } else  {
-
-                resolve->op_ret = -1;
-                resolve->op_errno = EINVAL;
-
-                server_resolve_all (frame);
-        }
-
-        return 0;
-}
-
-
-static int
-server_resolve_done (call_frame_t *frame)
-{
-        server_state_t    *state = NULL;
-        xlator_t          *bound_xl = NULL;
-
-        state = CALL_STATE (frame);
-        bound_xl = BOUND_XL (frame);
-
-        gf_server_print_request (frame);
-
-        state->resume_fn (frame, bound_xl);
-
-        return 0;
-}
-
-
-/*
- * This function is called multiple times, once per resolving one location/fd.
- * state->resolve_now is used to decide which location/fd is to be resolved now
- */
-static int
-server_resolve_all (call_frame_t *frame)
-{
-        server_state_t    *state = NULL;
-        xlator_t          *this = NULL;
-
-        this  = frame->this;
-        state = CALL_STATE (frame);
-
-        if (state->resolve_now == NULL) {
-
-                state->resolve_now = &state->resolve;
-                state->loc_now     = &state->loc;
-
-                server_resolve (frame);
-
-        } else if (state->resolve_now == &state->resolve) {
-
-                state->resolve_now = &state->resolve2;
-                state->loc_now     = &state->loc2;
-
-                server_resolve (frame);
-
-        } else if (state->resolve_now == &state->resolve2) {
-
-                server_resolve_done (frame);
-
-        } else {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid pointer for state->resolve_now");
-        }
-
-        return 0;
-}
-
-
-int
-gf_resolve_and_resume (call_frame_t *frame, server_resume_fn_t fn)
-{
-        server_state_t    *state = NULL;
-        xlator_t          *this  = NULL;
-
-        state = CALL_STATE (frame);
-        state->resume_fn = fn;
-
-        this = frame->this;
-
-        server_resolve_all (frame);
-
-        return 0;
-}
diff --git a/xlators/protocol/legacy/transport/Makefile.am b/xlators/protocol/legacy/transport/Makefile.am
deleted file mode 100644
index e2f97437c..000000000
--- a/xlators/protocol/legacy/transport/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = socket $(IBVERBS_SUBDIR)
-
-CLEANFILES = 
diff --git a/xlators/protocol/legacy/transport/ib-verbs/Makefile.am b/xlators/protocol/legacy/transport/ib-verbs/Makefile.am
deleted file mode 100644
index f963effea..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = src
-\ No newline at end of file
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/Makefile.am b/xlators/protocol/legacy/transport/ib-verbs/src/Makefile.am
deleted file mode 100644
index 3db7aff98..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-# TODO : need to change transportdir
-
-transport_LTLIBRARIES = ib-verbs.la
-transportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/transport
-
-ib_verbs_la_LDFLAGS = -module -avoidversion
-
-ib_verbs_la_SOURCES = ib-verbs.c name.c
-ib_verbs_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la     \
-	-libverbs $(top_builddir)/xlators/protocol/legacy/lib/src/libgfproto.la
-
-noinst_HEADERS = ib-verbs.h name.h ib-verbs-mem-types.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)  \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)   \
-	-I$(top_srcdir)/xlators/protocol/legacy/transport/ib-verbs                    \
-	-I$(top_srcdir)/xlators/protocol/legacy/lib/src
-
-CLEANFILES = *~
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs-mem-types.h b/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs-mem-types.h
deleted file mode 100644
index 80e81e56b..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs-mem-types.h
+++ /dev/null
@@ -1,39 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __IB_VERBS_MEM_TYPES_H__
-#define __IB_VERBS_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_ib_verbs_mem_types_ {
-        gf_ibv_mt_ib_verbs_private_t = gf_common_mt_end + 1,
-        gf_ibv_mt_ib_verbs_ioq_t,
-        gf_ibv_mt_transport_t,
-        gf_ibv_mt_ib_verbs_local_t,
-        gf_ibv_mt_ib_verbs_post_t,
-        gf_ibv_mt_char,
-        gf_ibv_mt_qpent,
-        gf_ibv_mt_ib_verbs_device_t,
-        gf_ibv_mt_end
-};
-#endif
-
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.c b/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.c
deleted file mode 100644
index 8b52fa884..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.c
+++ /dev/null
@@ -1,2625 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "dict.h"
-#include "glusterfs.h"
-#include "transport.h"
-#include "protocol.h"
-#include "logging.h"
-#include "xlator.h"
-#include "name.h"
-#include "ib-verbs.h"
-#include <signal.h>
-
-int32_t
-gf_resolve_ip6 (const char *hostname, 
-                uint16_t port, 
-                int family, 
-                void **dnscache, 
-                struct addrinfo **addr_info);
-
-static uint16_t 
-ib_verbs_get_local_lid (struct ibv_context *context,
-                        int32_t port)
-{
-        struct ibv_port_attr attr;
-
-        if (ibv_query_port (context, port, &attr))
-                return 0;
-
-        return attr.lid;
-}
-
-static const char *
-get_port_state_str(enum ibv_port_state pstate)
-{
-	switch (pstate) {
-	case IBV_PORT_DOWN:          return "PORT_DOWN";
-	case IBV_PORT_INIT:          return "PORT_INIT";
-	case IBV_PORT_ARMED:         return "PORT_ARMED";
-	case IBV_PORT_ACTIVE:        return "PORT_ACTIVE";
-	case IBV_PORT_ACTIVE_DEFER:  return "PORT_ACTIVE_DEFER";
-	default:                     return "invalid state";
-	}
-}
-
-static int32_t
-ib_check_active_port (struct ibv_context *ctx, uint8_t port)
-{
-        struct ibv_port_attr port_attr;
-
-        int32_t ret           = 0;
-        const char *state_str = NULL;
-
-        if (!ctx) {
-		gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "Error in supplied context");
-                return -1;
-	}
-
-        ret = ibv_query_port (ctx, port, &port_attr);
-
-        if (ret) {
-                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "Failed to query port %u properties", port);
-                return -1;
-        }
-
-        state_str = get_port_state_str (port_attr.state);
-        gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-                "Infiniband PORT: (%u) STATE: (%s)",
-                port, state_str);
-
-        if (port_attr.state == IBV_PORT_ACTIVE)
-                return 0;
-
-	return -1;
-}
-
-static int32_t
-ib_get_active_port (struct ibv_context *ib_ctx)
-{
-	struct ibv_device_attr ib_device_attr;
-
-	int32_t ret     = -1;
-	uint8_t ib_port = 0;
-
-	if (!ib_ctx) {
-		gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "Error in supplied context");
-		return -1;
-	}
-	if (ibv_query_device (ib_ctx, &ib_device_attr)) {
-		gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "Failed to query device properties");
-		return -1;
-	}
-
-	for (ib_port = 1; ib_port <= ib_device_attr.phys_port_cnt; ++ib_port) {
-                ret = ib_check_active_port (ib_ctx, ib_port);
-                if (ret == 0)
-                        return ib_port;
-
-                gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-                        "Port:(%u) not active", ib_port);
-                continue;
-	}
-	return ret;
-}
-
-
-
-static void
-ib_verbs_put_post (ib_verbs_queue_t *queue,
-                   ib_verbs_post_t *post)
-{
-        pthread_mutex_lock (&queue->lock);
-        if (post->prev) {
-                queue->active_count--;
-                post->prev->next = post->next;
-        }
-        if (post->next)
-                post->next->prev = post->prev;
-        post->prev = &queue->passive_posts;
-        post->next = post->prev->next;
-        post->prev->next = post;
-        post->next->prev = post;
-        queue->passive_count++;
-        pthread_mutex_unlock (&queue->lock);
-}
-
-
-static ib_verbs_post_t *
-ib_verbs_new_post (ib_verbs_device_t *device, int32_t len)
-{
-        ib_verbs_post_t *post;
-
-        post = (ib_verbs_post_t *) GF_CALLOC (1, sizeof (*post),
-                                   gf_ibv_mt_ib_verbs_post_t);
-        if (!post)
-                return NULL;
-
-        post->buf_size = len;
-
-        post->buf = valloc (len);
-        if (!post->buf) {
-                GF_FREE (post);
-                return NULL;
-        }
-
-        post->mr = ibv_reg_mr (device->pd,
-                               post->buf,
-                               post->buf_size,
-                               IBV_ACCESS_LOCAL_WRITE);
-        if (!post->mr) {
-                free (post->buf);
-                GF_FREE (post);
-                return NULL;
-        }
-
-        return post;
-}
-
-
-static ib_verbs_post_t *
-ib_verbs_get_post (ib_verbs_queue_t *queue)
-{
-        ib_verbs_post_t *post;
-
-        pthread_mutex_lock (&queue->lock);
-        {
-                post = queue->passive_posts.next;
-                if (post == &queue->passive_posts)
-                        post = NULL;
-    
-                if (post) {
-                        if (post->prev)
-                                post->prev->next = post->next;
-                        if (post->next)
-                                post->next->prev = post->prev;
-                        post->prev = &queue->active_posts;
-                        post->next = post->prev->next;
-                        post->prev->next = post;
-                        post->next->prev = post;
-                        post->reused++;
-                        queue->active_count++;
-                }
-        }
-        pthread_mutex_unlock (&queue->lock);
-
-        return post;
-}
-
-void
-ib_verbs_destroy_post (ib_verbs_post_t *post)
-{
-        ibv_dereg_mr (post->mr);
-        free (post->buf);
-        GF_FREE (post);
-}
-
-
-static int32_t
-__ib_verbs_quota_get (ib_verbs_peer_t *peer)
-{
-        int32_t ret = -1;
-        ib_verbs_private_t *priv = peer->trans->private;
-
-        if (priv->connected && peer->quota > 0) {
-                ret = peer->quota--;
-        }
-
-        return ret;
-}
-
-/*
-  static int32_t
-  ib_verbs_quota_get (ib_verbs_peer_t *peer)
-  {
-  int32_t ret = -1;
-  ib_verbs_private_t *priv = peer->trans->private;
-
-  pthread_mutex_lock (&priv->write_mutex);
-  {
-  ret = __ib_verbs_quota_get (peer);
-  }
-  pthread_mutex_unlock (&priv->write_mutex);
-
-  return ret;
-  }
-*/
-
-static void 
-__ib_verbs_ioq_entry_free (ib_verbs_ioq_t *entry)
-{
-        list_del_init (&entry->list);
-        if (entry->iobref)
-                iobref_unref (entry->iobref);
-
-        /* TODO: use mem-pool */
-        GF_FREE (entry->buf);
-
-        /* TODO: use mem-pool */
-        GF_FREE (entry);
-}
-
-
-static void
-__ib_verbs_ioq_flush (ib_verbs_peer_t *peer)
-{
-        ib_verbs_ioq_t *entry = NULL, *dummy = NULL;
-
-        list_for_each_entry_safe (entry, dummy, &peer->ioq, list) {
-                __ib_verbs_ioq_entry_free (entry);
-        }
-}
-
-
-static int32_t
-__ib_verbs_disconnect (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        int32_t ret = 0;
-
-        if (priv->connected || priv->tcp_connected) {
-                fcntl (priv->sock, F_SETFL, O_NONBLOCK);
-                if (shutdown (priv->sock, SHUT_RDWR) != 0) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_DEBUG,
-                                "shutdown () - error: %s",
-                                strerror (errno));
-                        ret = -errno;
-                        priv->tcp_connected = 0;
-                }
-        }
-  
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_post_send (struct ibv_qp *qp,
-                    ib_verbs_post_t *post,
-                    int32_t len)
-{
-        struct ibv_sge list = {
-                .addr = (unsigned long) post->buf,
-                .length = len,
-                .lkey = post->mr->lkey
-        };
-
-        struct ibv_send_wr wr = {
-                .wr_id      = (unsigned long) post,
-                .sg_list    = &list,
-                .num_sge    = 1,
-                .opcode     = IBV_WR_SEND,
-                .send_flags = IBV_SEND_SIGNALED,
-        }, *bad_wr;
-
-        if (!qp)
-                return -1;
-
-        return ibv_post_send (qp, &wr, &bad_wr);
-}
-
-
-static int32_t
-__ib_verbs_ioq_churn_entry (ib_verbs_peer_t *peer, ib_verbs_ioq_t *entry)
-{
-        int32_t ret = 0, quota = 0;
-        ib_verbs_private_t *priv = peer->trans->private;
-        ib_verbs_device_t *device = priv->device;
-        ib_verbs_options_t *options = &priv->options;
-        ib_verbs_post_t *post = NULL;
-        int32_t len = 0;
-
-        quota = __ib_verbs_quota_get (peer);
-        if (quota > 0) {
-                post = ib_verbs_get_post (&device->sendq);
-                if (!post) 
-                        post = ib_verbs_new_post (device, 
-                                                  (options->send_size + 2048));
-
-                len = iov_length ((const struct iovec *)&entry->vector, 
-                                  entry->count);
-                if  (len >= (options->send_size + 2048)) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "increase value of option 'transport.ib-verbs."
-                                "work-request-send-size' (given=> %"PRId64") "
-                                "to send bigger (%d) messages", 
-                                (options->send_size + 2048), len);
-                        return -1;
-                }
-
-                iov_unload (post->buf, 
-                            (const struct iovec *)&entry->vector, 
-                            entry->count);
-
-                ret = ib_verbs_post_send (peer->qp, post, len);
-                if (!ret) {
-                        __ib_verbs_ioq_entry_free (entry);
-                        ret = len;
-                } else {
-                        gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                                "ibv_post_send failed with ret = %d", ret);
-                        ib_verbs_put_post (&device->sendq, post);
-                        __ib_verbs_disconnect (peer->trans);
-                        ret = -1;
-                } 
-        }
-
-        return ret;
-}
-
-
-static int32_t
-__ib_verbs_ioq_churn (ib_verbs_peer_t *peer)
-{
-        ib_verbs_ioq_t *entry = NULL;
-        int32_t ret = 0;
-
-        while (!list_empty (&peer->ioq))
-        {
-                /* pick next entry */
-                entry = peer->ioq_next;
-
-                ret = __ib_verbs_ioq_churn_entry (peer, entry);
-
-                if (ret <= 0)
-                        break;
-        }
-
-        /*
-          list_for_each_entry_safe (entry, dummy, &peer->ioq, list) {
-          ret = __ib_verbs_ioq_churn_entry (peer, entry);
-          if (ret <= 0) {
-          break;
-          }
-          }
-        */
-
-        return ret;
-}
-
-static int32_t
-__ib_verbs_quota_put (ib_verbs_peer_t *peer)
-{
-        int32_t ret;
-
-        peer->quota++;
-        ret = peer->quota;
-
-        if (!list_empty (&peer->ioq)) {
-                ret = __ib_verbs_ioq_churn (peer);
-        }
-
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_quota_put (ib_verbs_peer_t *peer)
-{
-        int32_t ret;
-        ib_verbs_private_t *priv = peer->trans->private;
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                ret = __ib_verbs_quota_put (peer);
-        }
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_post_recv (struct ibv_srq *srq,
-                    ib_verbs_post_t *post)
-{
-        struct ibv_sge list = {
-                .addr   = (unsigned long) post->buf,
-                .length = post->buf_size,
-                .lkey   = post->mr->lkey
-        };
-
-        struct ibv_recv_wr wr = {
-                .wr_id  = (unsigned long) post,
-                .sg_list = &list,
-                .num_sge = 1,
-        }, *bad_wr;
-
-        return ibv_post_srq_recv (srq, &wr, &bad_wr);
-}
-
-
-static int32_t
-ib_verbs_writev (transport_t *this,
-                 ib_verbs_ioq_t *entry)
-{
-        int32_t ret = 0, need_append = 1;
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_peer_t  *peer = NULL;
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                if (!priv->connected) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "ib-verbs is not connected to post a "
-                                "send request");
-                        ret = -1;
-                        goto unlock;
-                }
-
-                peer = &priv->peer;
-                if (list_empty (&peer->ioq)) {
-                        ret = __ib_verbs_ioq_churn_entry (peer, entry);
-                        if (ret != 0) {
-                                need_append = 0;
-                        }
-                }
-
-                if (need_append) {
-                        list_add_tail (&entry->list, &peer->ioq);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-        return ret;
-}
-
-
-static ib_verbs_ioq_t *
-ib_verbs_ioq_new (char *buf, int len, struct iovec *vector, 
-                  int count, struct iobref *iobref)
-{
-        ib_verbs_ioq_t *entry = NULL;
-
-        /* TODO: use mem-pool */
-        entry = GF_CALLOC (1, sizeof (*entry), gf_ibv_mt_ib_verbs_ioq_t);
-
-        GF_ASSERT (count <= (MAX_IOVEC-2));
-
-        entry->header.colonO[0] = ':';
-        entry->header.colonO[1] = 'O';
-        entry->header.colonO[2] = '\0';
-        entry->header.version   = 42;
-        entry->header.size1     = hton32 (len);
-        entry->header.size2     = hton32 (iov_length (vector, count));
-
-        entry->vector[0].iov_base = &entry->header;
-        entry->vector[0].iov_len  = sizeof (entry->header);
-        entry->count++;
-
-        entry->vector[1].iov_base = buf;
-        entry->vector[1].iov_len  = len;
-        entry->count++;
-
-        if (vector && count)
-        {
-                memcpy (&entry->vector[2], vector, sizeof (*vector) * count);
-                entry->count += count;
-        }
-
-        if (iobref)
-                entry->iobref = iobref_ref (iobref);
-
-        entry->buf = buf;
-
-        INIT_LIST_HEAD (&entry->list);
-
-        return entry;
-}
-
-
-static int32_t
-ib_verbs_submit (transport_t *this, char *buf, int32_t len,
-                 struct iovec *vector, int count, struct iobref *iobref)
-{
-        int32_t ret = 0;
-        ib_verbs_ioq_t *entry = NULL;
-  
-        entry = ib_verbs_ioq_new (buf, len, vector, count, iobref);
-        ret = ib_verbs_writev (this, entry);
-
-        if (ret > 0) {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-static int
-ib_verbs_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                  struct iobuf **iobuf_p)
-{
-        ib_verbs_private_t *priv = this->private;
-        /* TODO: return error if !priv->connected, check with locks */
-        /* TODO: boundry checks for data_ptr/offset */
-        char *copy_from = NULL;
-        ib_verbs_header_t *header = NULL;
-        uint32_t size1, size2, data_len = 0;
-        char *hdr = NULL;
-        struct iobuf *iobuf = NULL;
-        int32_t ret = 0;
-
-        pthread_mutex_lock (&priv->recv_mutex);
-        {
-/*
-  while (!priv->data_ptr)
-  pthread_cond_wait (&priv->recv_cond, &priv->recv_mutex);
-*/
-
-                copy_from = priv->data_ptr + priv->data_offset;
-
-                priv->data_ptr = NULL;
-                data_len = priv->data_len;
-                pthread_cond_broadcast (&priv->recv_cond);
-        }
-        pthread_mutex_unlock (&priv->recv_mutex);
-
-        header = (ib_verbs_header_t *)copy_from;
-        if (strcmp (header->colonO, ":O")) {
-                gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                        "%s: corrupt header received", this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        size1 = ntoh32 (header->size1);
-        size2 = ntoh32 (header->size2);
-
-        if (data_len != (size1 + size2 + sizeof (*header))) {
-                gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                        "%s: sizeof data read from transport is not equal "
-                        "to the size specified in the header",
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-                  
-        copy_from += sizeof (*header);
-
-        if (size1) {
-                hdr = GF_CALLOC (1, size1, gf_ibv_mt_char);
-                if (!hdr) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unable to allocate header for peer %s",
-                                this->peerinfo.identifier);
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                memcpy (hdr, copy_from, size1);
-                copy_from += size1;
-                *hdr_p = hdr;
-        }
-        *hdrlen_p = size1;
-
-        if (size2) {
-                iobuf = iobuf_get (this->xl->ctx->iobuf_pool);
-                if (!iobuf) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unable to allocate IO buffer for peer %s",
-                                this->peerinfo.identifier);
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                memcpy (iobuf->ptr, copy_from, size2);
-                *iobuf_p = iobuf;
-        }
-
-err:
-        return ret;
-}
-
-
-static void
-ib_verbs_destroy_cq (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_device_t *device = priv->device;
-  
-        if (device->recv_cq)
-                ibv_destroy_cq (device->recv_cq);
-        device->recv_cq = NULL;
-  
-        if (device->send_cq)
-                ibv_destroy_cq (device->send_cq);
-        device->send_cq = NULL;
-
-        return;
-}
-
-
-static int32_t
-ib_verbs_create_cq (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        ib_verbs_device_t *device = priv->device;
-        int32_t ret = 0;
-
-        device->recv_cq = ibv_create_cq (priv->device->context,
-                                         options->recv_count * 2,
-                                         device,
-                                         device->recv_chan,
-                                         0);
-        if (!device->recv_cq) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_ERROR,
-                        "%s: creation of CQ failed",
-                        this->xl->name);
-                ret = -1;
-        } else if (ibv_req_notify_cq (device->recv_cq, 0)) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_ERROR,
-                        "%s: ibv_req_notify_cq on CQ failed",
-                        this->xl->name);
-                ret = -1;
-        }
-    
-        do {
-                /* TODO: make send_cq size dynamically adaptive */
-                device->send_cq = ibv_create_cq (priv->device->context,
-                                                 options->send_count * 1024,
-                                                 device,
-                                                 device->send_chan,
-                                                 0);
-                if (!device->send_cq) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_ERROR,
-                                "%s: creation of send_cq failed",
-                                this->xl->name);
-                        ret = -1;
-                        break;
-                }
-
-                if (ibv_req_notify_cq (device->send_cq, 0)) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_ERROR,
-                                "%s: ibv_req_notify_cq on send_cq failed",
-                                this->xl->name);
-                        ret = -1;
-                        break;
-                }
-        } while (0);
-
-        if (ret != 0)
-                ib_verbs_destroy_cq (this);
-
-        return ret;
-}
-
-
-static void
-ib_verbs_register_peer (ib_verbs_device_t *device,
-                        int32_t qp_num,
-                        ib_verbs_peer_t *peer)
-{
-        struct _qpent *ent;
-        ib_verbs_qpreg_t *qpreg = &device->qpreg;
-        int32_t hash = qp_num % 42;
-
-        pthread_mutex_lock (&qpreg->lock);
-        ent = qpreg->ents[hash].next;
-        while ((ent != &qpreg->ents[hash]) && (ent->qp_num != qp_num))
-                ent = ent->next;
-        if (ent->qp_num == qp_num) {
-                pthread_mutex_unlock (&qpreg->lock);
-                return;
-        }
-        ent = (struct _qpent *) GF_CALLOC (1, sizeof (*ent), gf_ibv_mt_qpent);
-        if (!ent)
-                return;
-        /* TODO: ref reg->peer */
-        ent->peer = peer;
-        ent->next = &qpreg->ents[hash];
-        ent->prev = ent->next->prev;
-        ent->next->prev = ent;
-        ent->prev->next = ent;
-        ent->qp_num = qp_num;
-        qpreg->count++;
-        pthread_mutex_unlock (&qpreg->lock);
-}
-
-
-static void
-ib_verbs_unregister_peer (ib_verbs_device_t *device,
-                          int32_t qp_num)
-{
-        struct _qpent *ent;
-        ib_verbs_qpreg_t *qpreg = &device->qpreg;
-        int32_t hash = qp_num % 42;
-
-        pthread_mutex_lock (&qpreg->lock);
-        ent = qpreg->ents[hash].next;
-        while ((ent != &qpreg->ents[hash]) && (ent->qp_num != qp_num))
-                ent = ent->next;
-        if (ent->qp_num != qp_num) {
-                pthread_mutex_unlock (&qpreg->lock);
-                return;
-        }
-        ent->prev->next = ent->next;
-        ent->next->prev = ent->prev;
-        /* TODO: unref reg->peer */
-        GF_FREE (ent);
-        qpreg->count--;
-        pthread_mutex_unlock (&qpreg->lock);
-}
-
-
-static ib_verbs_peer_t *
-__ib_verbs_lookup_peer (ib_verbs_device_t *device, int32_t qp_num)
-{
-        struct _qpent    *ent   = NULL;
-        ib_verbs_peer_t  *peer  = NULL;
-        ib_verbs_qpreg_t *qpreg = NULL;
-        int32_t hash            = 0;
-
-        qpreg = &device->qpreg;
-        hash = qp_num % 42;
-        ent = qpreg->ents[hash].next;
-        while ((ent != &qpreg->ents[hash]) && (ent->qp_num != qp_num))
-                ent = ent->next;
-
-        if (ent != &qpreg->ents[hash]) {
-                peer = ent->peer;
-        }
-
-        return peer;
-}
-
-/*
-static ib_verbs_peer_t *
-ib_verbs_lookup_peer (ib_verbs_device_t *device,
-                      int32_t qp_num)
-{
-        ib_verbs_qpreg_t *qpreg = NULL;
-        ib_verbs_peer_t  *peer  = NULL;
- 
-        qpreg = &device->qpreg;
-        pthread_mutex_lock (&qpreg->lock);
-        {
-                peer = __ib_verbs_lookup_peer (device, qp_num);
-        }
-        pthread_mutex_unlock (&qpreg->lock);
-
-        return peer;
-}
-*/
-
-
-static void
-__ib_verbs_destroy_qp (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-
-        if (priv->peer.qp) {
-                ib_verbs_unregister_peer (priv->device, priv->peer.qp->qp_num);
-                ibv_destroy_qp (priv->peer.qp);
-        }
-        priv->peer.qp = NULL;
-
-        return;
-}
-
-
-static int32_t
-ib_verbs_create_qp (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        ib_verbs_device_t *device = priv->device;
-        int32_t ret = 0;
-        ib_verbs_peer_t *peer;
-
-        peer = &priv->peer;
-        struct ibv_qp_init_attr init_attr = {
-                .send_cq        = device->send_cq,
-                .recv_cq        = device->recv_cq,
-                .srq            = device->srq,
-                .cap            = {
-                        .max_send_wr  = peer->send_count,
-                        .max_recv_wr  = peer->recv_count,
-                        .max_send_sge = 1,
-                        .max_recv_sge = 1
-                },
-                .qp_type = IBV_QPT_RC
-        };
-  
-        struct ibv_qp_attr attr = {
-                .qp_state        = IBV_QPS_INIT,
-                .pkey_index      = 0,
-                .port_num        = options->port,
-                .qp_access_flags = 0
-        };
-  
-        peer->qp = ibv_create_qp (device->pd, &init_attr);
-        if (!peer->qp) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_CRITICAL,
-                        "%s: could not create QP",
-                        this->xl->name);
-                ret = -1;
-                goto out;
-        } else if (ibv_modify_qp (peer->qp, &attr,
-                                  IBV_QP_STATE              |
-                                  IBV_QP_PKEY_INDEX         |
-                                  IBV_QP_PORT               |
-                                  IBV_QP_ACCESS_FLAGS)) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_ERROR,
-                        "%s: failed to modify QP to INIT state",
-                        this->xl->name);
-                ret = -1;
-                goto out;
-        }
-
-        peer->local_lid = ib_verbs_get_local_lid (device->context,
-                                                  options->port);
-        peer->local_qpn = peer->qp->qp_num;
-        peer->local_psn = lrand48 () & 0xffffff;
-
-        ib_verbs_register_peer (device, peer->qp->qp_num, peer);
-
-out:
-        if (ret == -1)
-                __ib_verbs_destroy_qp (this);
-
-        return ret;
-}
-
-
-static void
-ib_verbs_destroy_posts (transport_t *this)
-{
-
-}
-
-
-static int32_t
-__ib_verbs_create_posts (transport_t *this,
-                         int32_t count,
-                         int32_t size,
-                         ib_verbs_queue_t *q)
-{
-        int32_t i;
-        int32_t ret = 0;
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_device_t *device = priv->device;
-
-        for (i=0 ; i<count ; i++) {
-                ib_verbs_post_t *post;
-
-                post = ib_verbs_new_post (device, size + 2048);
-                if (!post) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_ERROR,
-                                "%s: post creation failed",
-                                this->xl->name);
-                        ret = -1;
-                        break;
-                }
-
-                ib_verbs_put_post (q, post);
-        }
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_create_posts (transport_t *this)
-{
-        int32_t i, ret;
-        ib_verbs_post_t *post = NULL;
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        ib_verbs_device_t *device = priv->device;
-
-        ret =  __ib_verbs_create_posts (this, options->send_count,
-                                        options->send_size,
-                                        &device->sendq);
-        if (!ret)
-                ret =  __ib_verbs_create_posts (this, options->recv_count,
-                                                options->recv_size,
-                                                &device->recvq);
-
-        if (!ret) {
-                for (i=0 ; i<options->recv_count ; i++) {
-                        post = ib_verbs_get_post (&device->recvq);
-                        if (ib_verbs_post_recv (device->srq, post) != 0) {
-                                ret = -1;
-                                break;
-                        }
-                }
-        }
-
-        if (ret)
-                ib_verbs_destroy_posts (this);
-
-        return ret;
-}
-
-
-static int32_t
-ib_verbs_connect_qp (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        struct ibv_qp_attr attr = {
-                .qp_state               = IBV_QPS_RTR,
-                .path_mtu               = options->mtu,
-                .dest_qp_num            = priv->peer.remote_qpn,
-                .rq_psn                 = priv->peer.remote_psn,
-                .max_dest_rd_atomic     = 1,
-                .min_rnr_timer          = 12,
-                .ah_attr                = {
-                        .is_global      = 0,
-                        .dlid           = priv->peer.remote_lid,
-                        .sl             = 0,
-                        .src_path_bits  = 0,
-                        .port_num       = options->port
-                }
-        };
-        if (ibv_modify_qp (priv->peer.qp, &attr,
-                           IBV_QP_STATE              |
-                           IBV_QP_AV                 |
-                           IBV_QP_PATH_MTU           |
-                           IBV_QP_DEST_QPN           |
-                           IBV_QP_RQ_PSN             |
-                           IBV_QP_MAX_DEST_RD_ATOMIC |
-                           IBV_QP_MIN_RNR_TIMER)) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_CRITICAL,
-                        "Failed to modify QP to RTR\n");
-                return -1;
-        }
-
-        /* TODO: make timeout and retry_cnt configurable from options */
-        attr.qp_state       = IBV_QPS_RTS;
-        attr.timeout        = 14;
-        attr.retry_cnt      = 7;
-        attr.rnr_retry      = 7;
-        attr.sq_psn         = priv->peer.local_psn;
-        attr.max_rd_atomic  = 1;
-        if (ibv_modify_qp (priv->peer.qp, &attr,
-                           IBV_QP_STATE              |
-                           IBV_QP_TIMEOUT            |
-                           IBV_QP_RETRY_CNT          |
-                           IBV_QP_RNR_RETRY          |
-                           IBV_QP_SQ_PSN             |
-                           IBV_QP_MAX_QP_RD_ATOMIC)) {
-                gf_log ("transport/ib-verbs",
-                        GF_LOG_CRITICAL,
-                        "Failed to modify QP to RTS\n");
-                return -1;
-        }
-
-        return 0;
-}
-
-static int32_t
-__ib_verbs_teardown (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-
-        __ib_verbs_destroy_qp (this);
-
-        if (!list_empty (&priv->peer.ioq)) {
-                __ib_verbs_ioq_flush (&priv->peer);
-        }
-
-        /* TODO: decrement cq size */
-        return 0;
-}
-
-/*
- * return value:
- *   0 = success (completed)
- *  -1 = error
- * > 0 = incomplete
- */
-
-static int
-__tcp_rwv (transport_t *this, struct iovec *vector, int count,
-           struct iovec **pending_vector, int *pending_count,
-           int write)
-{
-        ib_verbs_private_t *priv = NULL;
-        int sock = -1;
-        int ret = -1;
-        struct iovec *opvector = vector;
-        int opcount = count;
-        int moved = 0;
-
-        priv = this->private;
-        sock = priv->sock;
-
-        while (opcount)
-        {
-                if (write)
-                {
-                        ret = writev (sock, opvector, opcount);
-
-                        if (ret == 0 || (ret == -1 && errno == EAGAIN))
-                        {
-                                /* done for now */
-                                break;
-                        }
-                }
-                else
-                {
-                        ret = readv (sock, opvector, opcount);
-
-                        if (ret == -1 && errno == EAGAIN)
-                        {
-                                /* done for now */
-                                break;
-                        }
-                }
-
-                if (ret == 0)
-                {
-                        gf_log (this->xl->name, GF_LOG_DEBUG, 
-                                "EOF from peer %s", this->peerinfo.identifier);
-                        opcount = -1;
-                        errno = ENOTCONN;
-                        break;
-                }
-
-                if (ret == -1)
-                {
-                        if (errno == EINTR)
-                                continue;
-
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "%s failed (%s)", write ? "writev" : "readv",
-                                strerror (errno));
-                        if (write && !priv->connected && 
-                            (errno == ECONNREFUSED))
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "possible mismatch of 'transport-type'"
-                                        " in protocol server and client. "
-                                        "check volume file");
-                        opcount = -1;
-                        break;
-                }
-
-                moved = 0;
-
-                while (moved < ret)
-                {
-                        if ((ret - moved) >= opvector[0].iov_len)
-                        {
-                                moved += opvector[0].iov_len;
-                                opvector++;
-                                opcount--;
-                        }
-                        else
-                        {
-                                opvector[0].iov_len -= (ret - moved);
-                                opvector[0].iov_base += (ret - moved);
-                                moved += (ret - moved);
-                        }
-                        while (opcount && !opvector[0].iov_len)
-                        {
-                                opvector++;
-                                opcount--;
-                        }
-                }
-        }
-
-        if (pending_vector)
-                *pending_vector = opvector;
-
-        if (pending_count)
-                *pending_count = opcount;
-
-        return opcount;
-}
-
-
-static int
-__tcp_readv (transport_t *this, struct iovec *vector, int count,
-             struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-
-        ret = __tcp_rwv (this, vector, count, 
-                         pending_vector, pending_count, 0);
-
-        return ret;
-}
-
-
-static int
-__tcp_writev (transport_t *this, struct iovec *vector, int count,
-              struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-        ib_verbs_private_t *priv = this->private;
-
-        ret = __tcp_rwv (this, vector, count, pending_vector, 
-                         pending_count, 1);
-
-        if (ret > 0) {
-                /* TODO: Avoid multiple calls when socket is already 
-                   registered for POLLOUT */
-                priv->idx = event_select_on (this->xl->ctx->event_pool, 
-                                             priv->sock, priv->idx, -1, 1);
-        } else if (ret == 0) {
-                priv->idx = event_select_on (this->xl->ctx->event_pool, 
-                                             priv->sock,
-                                             priv->idx, -1, 0);
-        }
-
-        return ret;
-}
-
-
-static void *
-ib_verbs_recv_completion_proc (void *data)
-{
-        struct ibv_comp_channel *chan = data;
-        ib_verbs_private_t      *priv = NULL;
-        ib_verbs_device_t       *device;
-        ib_verbs_post_t         *post;
-        ib_verbs_peer_t         *peer;
-        struct ibv_cq           *event_cq;
-        struct ibv_wc            wc;
-        void                    *event_ctx;
-        int32_t                  ret  = 0;
-
-
-        while (1) {
-                ret = ibv_get_cq_event (chan, &event_cq, &event_ctx);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "ibv_get_cq_event failed, terminating recv "
-                                "thread %d (%d)", ret, errno);
-                        continue;
-                }
-
-                device = event_ctx;
-    
-                ret = ibv_req_notify_cq (event_cq, 0);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "ibv_req_notify_cq on %s failed, terminating "
-                                "recv thread: %d (%d)",
-                                device->device_name, ret, errno);
-                        continue;
-                }
-
-                device = (ib_verbs_device_t *) event_ctx;
-
-                while ((ret = ibv_poll_cq (event_cq, 1, &wc)) > 0) {
-                        post = (ib_verbs_post_t *) (long) wc.wr_id;
-
-                        pthread_mutex_lock (&device->qpreg.lock);
-                        {
-                                peer = __ib_verbs_lookup_peer (device,
-                                                               wc.qp_num);
-
-                                /*
-                                 * keep a refcount on transport so that it
-                                 * doesnot get freed because of some error
-                                 * indicated by wc.status till we are done
-                                 * with usage of peer and thereby that of trans.
-                                 */
-                                if (peer != NULL) {
-                                        transport_ref (peer->trans);
-                                }
-                        }
-                        pthread_mutex_unlock (&device->qpreg.lock);
-
-                        if (wc.status != IBV_WC_SUCCESS) {
-                                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                        "recv work request on `%s' returned "
-                                        "error (%d)",
-                                        device->device_name,
-                                        wc.status);
-                                if (peer) {
-                                        transport_unref (peer->trans);
-                                        transport_disconnect (peer->trans);
-                                }
-
-                                if (post) {
-                                        ib_verbs_post_recv (device->srq, post);
-                                }
-                                continue;
-                        }
-
-                        if (peer) {
-                                priv = peer->trans->private;
-        
-                                pthread_mutex_lock (&priv->recv_mutex);
-                                {
-                                        while (priv->data_ptr)
-                                                pthread_cond_wait (&priv->recv_cond,
-                                                                   &priv->recv_mutex);
-          
-                                        priv->data_ptr = post->buf;
-                                        priv->data_offset = 0;
-                                        priv->data_len = wc.byte_len;
-          
-                                        /*pthread_cond_broadcast (&priv->recv_cond);*/
-                                }
-                                pthread_mutex_unlock (&priv->recv_mutex);
-        
-                                if ((ret = xlator_notify (peer->trans->xl, GF_EVENT_POLLIN, 
-                                                          peer->trans, NULL)) == -1) {
-                                        gf_log ("transport/ib-verbs",
-                                                GF_LOG_DEBUG, 
-                                                "pollin notification to %s "
-                                                "failed, disconnecting "
-                                                "transport", 
-                                                peer->trans->xl->name);
-                                        transport_disconnect (peer->trans);
-                                }
-
-                                transport_unref (peer->trans);
-                        } else {
-                                gf_log ("transport/ib-verbs",
-                                        GF_LOG_DEBUG,
-                                        "could not lookup peer for qp_num: %d",
-                                        wc.qp_num);
-                        }
-                        ib_verbs_post_recv (device->srq, post);
-                }
-    
-                if (ret < 0) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_ERROR,
-                                "ibv_poll_cq on `%s' returned error "
-                                "(ret = %d, errno = %d)",
-                                device->device_name, ret, errno);
-                        continue;
-                }
-                ibv_ack_cq_events (event_cq, 1);
-        }
-        return NULL;
-}
-
-
-static void *
-ib_verbs_send_completion_proc (void *data)
-{
-        struct ibv_comp_channel *chan = data;
-        ib_verbs_post_t *post;
-        ib_verbs_peer_t *peer;
-        struct ibv_cq *event_cq;
-        void *event_ctx;
-        ib_verbs_device_t *device;
-        struct ibv_wc wc;
-        int32_t ret;
-        
-        while (1) {
-                ret = ibv_get_cq_event (chan, &event_cq, &event_ctx);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "ibv_get_cq_event on failed, terminating "
-                                "send thread: %d (%d)", ret, errno);
-                        continue;
-                }
-       
-                device = event_ctx;
-
-                ret = ibv_req_notify_cq (event_cq, 0);
-                if (ret) {
-                        gf_log ("transport/ib-verbs",  GF_LOG_ERROR,
-                                "ibv_req_notify_cq on %s failed, terminating "
-                                "send thread: %d (%d)",
-                                device->device_name, ret, errno);
-                        continue;
-                }
-
-                while ((ret = ibv_poll_cq (event_cq, 1, &wc)) > 0) {
-                        post = (ib_verbs_post_t *) (long) wc.wr_id;
-
-                        pthread_mutex_lock (&device->qpreg.lock);
-                        {
-                                peer = __ib_verbs_lookup_peer (device,
-                                                               wc.qp_num);
-
-                                /*
-                                 * keep a refcount on transport so that it
-                                 * doesnot get freed because of some error
-                                 * indicated by wc.status till we are done
-                                 * with usage of peer and thereby that of trans.
-                                 */
-                                if (peer != NULL) {
-                                        transport_ref (peer->trans);
-                                }
-                        }
-                        pthread_mutex_unlock (&device->qpreg.lock);
-
-                        if (wc.status != IBV_WC_SUCCESS) {
-                                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                        "send work request on `%s' returned "
-                                        "error wc.status = %d, wc.vendor_err "
-                                        "= %d, post->buf = %p, wc.byte_len = "
-                                        "%d, post->reused = %d",
-                                        device->device_name, wc.status, 
-                                        wc.vendor_err,
-                                        post->buf, wc.byte_len, post->reused);
-                                if (wc.status == IBV_WC_RETRY_EXC_ERR)
-                                        gf_log ("ib-verbs", GF_LOG_ERROR,
-                                                "connection between client and"
-                                                " server not working. check by"
-                                                " running 'ibv_srq_pingpong'. "
-                                                "also make sure subnet manager"
-                                                " is running (eg: 'opensm'), "
-                                                "or check if ib-verbs port is "
-                                                "valid (or active) by running "
-                                                " 'ibv_devinfo'. contact "
-                                                "Gluster Support Team if "
-                                                "the problem persists.");
-                                if (peer)
-                                        transport_disconnect (peer->trans);
-                        }
-
-                        if (post) {
-                                ib_verbs_put_post (&device->sendq, post);
-                        }
-      
-                        if (peer) {
-                                int quota_ret = ib_verbs_quota_put (peer);
-                                if (quota_ret < 0) {
-                                        gf_log ("ib-verbs", GF_LOG_DEBUG,
-                                                "failed to send message");
-                                        
-                                }
-
-                                transport_unref (peer->trans);
-                        } else {
-                                gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                                        "could not lookup peer for qp_num: %d",
-                                        wc.qp_num);
-                        }
-                }
-
-                if (ret < 0) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "ibv_poll_cq on `%s' returned error (ret = %d,"
-                                " errno = %d)",
-                                device->device_name, ret, errno);
-                        continue;
-                }
-                ibv_ack_cq_events (event_cq, 1); 
-        }
-
-        return NULL;
-}
-
-static void
-ib_verbs_options_init (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        int32_t mtu;
-        data_t *temp;
-
-        /* TODO: validate arguments from options below */
-
-        options->send_size = this->xl->ctx->page_size * 4; /* 512 KB */
-        options->recv_size = this->xl->ctx->page_size * 4; /* 512 KB */
-        options->send_count = 32;
-        options->recv_count = 32;
-
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.work-request-send-count");
-        if (temp)
-                options->send_count = data_to_int32 (temp);
-
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.work-request-recv-count");
-        if (temp)
-                options->recv_count = data_to_int32 (temp);
-
-        options->port = 0;
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.port");
-        if (temp)
-                options->port = data_to_uint64 (temp);
-
-        options->mtu = mtu = IBV_MTU_2048;
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.mtu");
-        if (temp)
-                mtu = data_to_int32 (temp);
-        switch (mtu) {
-        case 256: options->mtu = IBV_MTU_256;
-                break;
-        case 512: options->mtu = IBV_MTU_512;
-                break;
-        case 1024: options->mtu = IBV_MTU_1024;
-                break;
-        case 2048: options->mtu = IBV_MTU_2048;
-                break;
-        case 4096: options->mtu = IBV_MTU_4096;
-                break;
-        default:
-                if (temp)
-                        gf_log ("transport/ib-verbs", GF_LOG_WARNING,
-                                "%s: unrecognized MTU value '%s', defaulting "
-                                "to '2048'", this->xl->name,
-                                data_to_str (temp));
-                else
-                        gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-                                "%s: defaulting MTU to '2048'",
-                                this->xl->name);
-                options->mtu = IBV_MTU_2048;
-                break;
-        }
-
-        temp = dict_get (this->xl->options,
-                         "transport.ib-verbs.device-name");
-        if (temp)
-                options->device_name = gf_strdup (temp->data);
-
-        return;
-}
-
-static void
-ib_verbs_queue_init (ib_verbs_queue_t *queue)
-{
-        pthread_mutex_init (&queue->lock, NULL);
-
-        queue->active_posts.next = &queue->active_posts;
-        queue->active_posts.prev = &queue->active_posts;
-        queue->passive_posts.next = &queue->passive_posts;
-        queue->passive_posts.prev = &queue->passive_posts;
-}
-
-
-static ib_verbs_device_t *
-ib_verbs_get_device (transport_t *this,
-		     struct ibv_context *ibctx)
-{
-        glusterfs_ctx_t *ctx        = this->xl->ctx;
-        ib_verbs_private_t *priv    = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        char *device_name           = priv->options.device_name;
-        uint32_t port               = priv->options.port;
-
-        uint8_t active_port = 0;
-        int32_t ret         = 0;
-        int32_t i           = 0;
-
-        ib_verbs_device_t *trav;
-
-        trav = ctx->ib;
-        while (trav) {
-                if ((!strcmp (trav->device_name, device_name)) && 
-                    (trav->port == port))
-                        break;
-                trav = trav->next;
-        }
-
-        if (!trav) {
-
-                trav = GF_CALLOC (1, sizeof (*trav), 
-                                  gf_ibv_mt_ib_verbs_device_t);
-                if (!trav)
-                        return NULL;
-                priv->device = trav;
-
-                trav->context = ibctx;
-
-		ret = ib_get_active_port (trav->context);
-
-		if (ret < 0) {
-			if (!port) {
-				gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-					"Failed to find any active ports and "
-					"none specified in volume file,"
-                                        " exiting");
-				return NULL;
-			}
-		}
-
-		active_port = ret;
-
-                if (port) {
-                        ret = ib_check_active_port (trav->context, port);
-                        if (ret < 0) {
-                                gf_log ("transport/ib-verbs", GF_LOG_WARNING,
-                                        "On device %s: provided port:%u is "
-                                        "found to be offline, continuing to "
-                                        "use the same port", device_name, port);
-                        }
-		} else {
-			priv->options.port = active_port;
-			port = active_port;
-			gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-				"Port unspecified in volume file using active "
-                                "port: %u", port);
-                }
-
-                trav->device_name = gf_strdup (device_name);
-                trav->port = port;
-
-                trav->next = ctx->ib;
-                ctx->ib = trav;
-
-                trav->send_chan = ibv_create_comp_channel (trav->context);
-                if (!trav->send_chan) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not create send completion channel",
-                                device_name);
-                        /* TODO: cleanup current mess */
-                        return NULL;
-                }
-    
-                trav->recv_chan = ibv_create_comp_channel (trav->context);
-                if (!trav->recv_chan) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "could not create recv completion channel");
-                        /* TODO: cleanup current mess */
-                        return NULL;
-                }
-      
-                if (ib_verbs_create_cq (this) < 0) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not create CQ",
-                                this->xl->name);
-                        return NULL;
-                }
-
-                /* protection domain */
-                trav->pd = ibv_alloc_pd (trav->context);
-
-                if (!trav->pd) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not allocate protection domain",
-                                this->xl->name);
-                        return NULL;
-                }
-
-                struct ibv_srq_init_attr attr = {
-                        .attr = {
-                                .max_wr = options->recv_count,
-                                .max_sge = 1
-                        }
-                };
-                trav->srq = ibv_create_srq (trav->pd, &attr);
-
-                if (!trav->srq) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not create SRQ",
-                                this->xl->name);
-                        return NULL;
-                }
-
-                /* queue init */
-                ib_verbs_queue_init (&trav->sendq);
-                ib_verbs_queue_init (&trav->recvq);
-
-                if (ib_verbs_create_posts (this) < 0) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: could not allocate posts",
-                                this->xl->name);
-                        return NULL;
-                }
-
-                /* completion threads */
-                ret = pthread_create (&trav->send_thread,
-                                      NULL,
-                                      ib_verbs_send_completion_proc,
-                                      trav->send_chan);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "could not create send completion thread");
-                        return NULL;
-                }
-                ret = pthread_create (&trav->recv_thread,
-                                      NULL,
-                                      ib_verbs_recv_completion_proc,
-                                      trav->recv_chan);
-                if (ret) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "could not create recv completion thread");
-                        return NULL;
-                }
-  
-                /* qpreg */
-                pthread_mutex_init (&trav->qpreg.lock, NULL);
-                for (i=0; i<42; i++) {
-                        trav->qpreg.ents[i].next = &trav->qpreg.ents[i];
-                        trav->qpreg.ents[i].prev = &trav->qpreg.ents[i];
-                }
-        }
-        return trav;
-}
-
-static int32_t 
-ib_verbs_init (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = &priv->options;
-        struct ibv_device **dev_list;
-	struct ibv_context *ib_ctx = NULL;
-	int32_t ret = 0;
-
-        ib_verbs_options_init (this);
-
-        {
-                dev_list = ibv_get_device_list (NULL);
-
-		if (!dev_list) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_CRITICAL,
-                                "Failed to get IB devices");
-			ret = -1;
-			goto cleanup;
-                }
-
-                if (!*dev_list) {
-                        gf_log ("transport/ib-verbs",
-                                GF_LOG_CRITICAL,
-                                "No IB devices found");
-			ret = -1;
-                        goto cleanup;
-                }
-
-                if (!options->device_name) {
-                        if (*dev_list) {
-                                options->device_name = 
-                                        gf_strdup (ibv_get_device_name (*dev_list));
-                        } else {
-                                gf_log ("transport/ib-verbs", GF_LOG_CRITICAL,
-                                        "IB device list is empty. Check for "
-                                        "'ib_uverbs' module");
-                                return -1;
-                                goto cleanup;
-                        }
-                }
-
-		while (*dev_list) {
-                        if (!strcmp (ibv_get_device_name (*dev_list),
-                                     options->device_name)) {
-                                ib_ctx = ibv_open_device (*dev_list);
-
-                                if (!ib_ctx) {
-                                        gf_log ("transport/ib-verbs",
-                                                GF_LOG_ERROR,
-                                                "Failed to get infiniband"
-                                                "device context");
-                                        ret = -1;
-                                        goto cleanup;
-                                }
-                                break;
-                        }
-			++dev_list;
-		}
-
-		priv->device = ib_verbs_get_device (this, ib_ctx);
-
-                if (!priv->device) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "could not create ib_verbs device for %s",
-                                options->device_name);
-                        ret = -1;
-			goto cleanup;
-                }
-        }
-
-        priv->peer.trans = this;
-        INIT_LIST_HEAD (&priv->peer.ioq);
-  
-        pthread_mutex_init (&priv->read_mutex, NULL);
-        pthread_mutex_init (&priv->write_mutex, NULL);
-        pthread_mutex_init (&priv->recv_mutex, NULL);
-        pthread_cond_init (&priv->recv_cond, NULL);
-
-cleanup:
-	if (-1 == ret) {
-		if (ib_ctx)
-			ibv_close_device (ib_ctx);
-	}
-
-	if (dev_list)
-		ibv_free_device_list (dev_list);
-
-	return ret;
-}
-
-
-static int32_t
-ib_verbs_disconnect (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        int32_t ret = 0;
- 
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                ret = __ib_verbs_disconnect (this);
-        }
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        return ret;
-}
-
-
-static int32_t
-__tcp_connect_finish (int fd)
-{
-        int ret = -1;
-        int optval = 0;
-        socklen_t optlen = sizeof (int);
-
-        ret = getsockopt (fd, SOL_SOCKET, SO_ERROR,
-                          (void *)&optval, &optlen);
-
-        if (ret == 0 && optval)
-        {
-                errno = optval;
-                ret = -1;
-        }
-
-        return ret;
-}
-
-static inline void
-ib_verbs_fill_handshake_data (char *buf, struct ib_verbs_nbio *nbio, 
-                              ib_verbs_private_t *priv)
-{
-        sprintf (buf,
-                 "QP1:RECV_BLKSIZE=%08x:SEND_BLKSIZE=%08x\n"
-                 "QP1:LID=%04x:QPN=%06x:PSN=%06x\n",
-                 priv->peer.recv_size,
-                 priv->peer.send_size,
-                 priv->peer.local_lid,
-                 priv->peer.local_qpn,
-                 priv->peer.local_psn);
-
-        nbio->vector.iov_base = buf;
-        nbio->vector.iov_len = strlen (buf) + 1;
-        nbio->count = 1;
-        return;
-}
-
-static inline void
-ib_verbs_fill_handshake_ack (char *buf, struct ib_verbs_nbio *nbio)
-{
-        sprintf (buf, "DONE\n");
-        nbio->vector.iov_base = buf;
-        nbio->vector.iov_len = strlen (buf) + 1;
-        nbio->count = 1;
-        return;
-}
-
-static int
-ib_verbs_handshake_pollin (transport_t *this)
-{
-        int ret = 0;
-        ib_verbs_private_t *priv = this->private;
-        char *buf = priv->handshake.incoming.buf;
-        int32_t recv_buf_size, send_buf_size;
-        socklen_t sock_len;
-
-        if (priv->handshake.incoming.state == IB_VERBS_HANDSHAKE_COMPLETE) {
-                return -1;
-        }
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                while (priv->handshake.incoming.state != IB_VERBS_HANDSHAKE_COMPLETE)
-                {
-                        switch (priv->handshake.incoming.state) 
-                        {
-                        case IB_VERBS_HANDSHAKE_START:
-                                buf = priv->handshake.incoming.buf = GF_CALLOC (1, 256, gf_ibv_mt_char);
-                                ib_verbs_fill_handshake_data (buf, &priv->handshake.incoming, priv);
-                                buf[0] = 0;
-                                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_RECEIVING_DATA;
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_RECEIVING_DATA:
-                                ret = __tcp_readv (this, 
-                                                   &priv->handshake.incoming.vector, 
-                                                   priv->handshake.incoming.count,
-                                                   &priv->handshake.incoming.pending_vector, 
-                                                   &priv->handshake.incoming.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                                "partial header read on NB socket. continue later");
-                                        goto unlock;
-                                }
-            
-                                if (!ret) {
-                                        priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_RECEIVED_DATA;
-                                }
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_RECEIVED_DATA:
-                                ret = sscanf (buf,
-                                              "QP1:RECV_BLKSIZE=%08x:SEND_BLKSIZE=%08x\n"
-                                              "QP1:LID=%04x:QPN=%06x:PSN=%06x\n",
-                                              &recv_buf_size,
-                                              &send_buf_size,
-                                              &priv->peer.remote_lid,
-                                              &priv->peer.remote_qpn,
-                                              &priv->peer.remote_psn);
-
-                                if ((ret != 5) && (strncmp (buf, "QP1:", 4))) {
-                                        gf_log ("transport/ib-verbs", 
-                                                GF_LOG_CRITICAL,
-                                                "%s: remote-host(%s)'s "
-                                                "transport type is different",
-                                                this->xl->name, 
-                                                this->peerinfo.identifier);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-
-                                if (recv_buf_size < priv->peer.recv_size)
-                                        priv->peer.recv_size = recv_buf_size;
-                                if (send_buf_size < priv->peer.send_size)
-                                        priv->peer.send_size = send_buf_size;
-          
-                                gf_log ("transport/ib-verbs", GF_LOG_TRACE,
-                                        "%s: transacted recv_size=%d "
-                                        "send_size=%d",
-                                        this->xl->name, priv->peer.recv_size,
-                                        priv->peer.send_size);
-
-                                priv->peer.quota = priv->peer.send_count;
-
-                                if (ib_verbs_connect_qp (this)) {
-                                        gf_log ("transport/ib-verbs", 
-                                                GF_LOG_ERROR,
-                                                "%s: failed to connect with "
-                                                "remote QP", this->xl->name);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-                                ib_verbs_fill_handshake_ack (buf, &priv->handshake.incoming);
-                                buf[0] = 0;
-                                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_RECEIVING_ACK;
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_RECEIVING_ACK:
-                                ret = __tcp_readv (this, 
-                                                   &priv->handshake.incoming.vector, 
-                                                   priv->handshake.incoming.count,
-                                                   &priv->handshake.incoming.pending_vector, 
-                                                   &priv->handshake.incoming.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                                "partial header read on NB "
-                                                "socket. continue later");
-                                        goto unlock;
-                                }
-            
-                                if (!ret) {
-                                        priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_RECEIVED_ACK;
-                                }
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_RECEIVED_ACK:
-                                if (strncmp (buf, "DONE", 4)) {
-                                        gf_log ("transport/ib-verbs", 
-                                                GF_LOG_DEBUG,
-                                                "%s: handshake-3 did not "
-                                                "return 'DONE' (%s)",
-                                                this->xl->name, buf);
-                                        ret = -1;
-                                        goto unlock;
-                                }
-                                ret = 0;
-                                priv->connected = 1;
-                                sock_len = sizeof (struct sockaddr_storage);
-                                getpeername (priv->sock,
-                                             (struct sockaddr *) &this->peerinfo.sockaddr,
-                                             &sock_len);
-
-                                GF_FREE (priv->handshake.incoming.buf);
-                                priv->handshake.incoming.buf = NULL;
-                                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_COMPLETE;
-                        }
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (ret == -1) {
-                transport_disconnect (this);
-        } else {
-                ret = 0;
-        }
-
-        if (!ret && priv->connected) {
-                ret = xlator_notify (this->xl, GF_EVENT_CHILD_UP, this);
-        }
-
-        return ret;
-}
-
-static int 
-ib_verbs_handshake_pollout (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        char *buf = priv->handshake.outgoing.buf;
-        int32_t ret = 0;
-
-        if (priv->handshake.outgoing.state == IB_VERBS_HANDSHAKE_COMPLETE) {
-                return 0;
-        }
-
-        pthread_mutex_unlock (&priv->write_mutex);
-        {
-                while (priv->handshake.outgoing.state != IB_VERBS_HANDSHAKE_COMPLETE)
-                {
-                        switch (priv->handshake.outgoing.state) 
-                        {
-                        case IB_VERBS_HANDSHAKE_START:
-                                buf = priv->handshake.outgoing.buf = GF_CALLOC (1, 256, gf_ibv_mt_char);
-                                ib_verbs_fill_handshake_data (buf, &priv->handshake.outgoing, priv);
-                                priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_SENDING_DATA;
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_SENDING_DATA:
-                                ret = __tcp_writev (this, 
-                                                    &priv->handshake.outgoing.vector, 
-                                                    priv->handshake.outgoing.count,
-                                                    &priv->handshake.outgoing.pending_vector, 
-                                                    &priv->handshake.outgoing.pending_count);
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                                "partial header read on NB socket. continue later");
-                                        goto unlock;
-                                }
-            
-                                if (!ret) {
-                                        priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_SENT_DATA;
-                                }
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_SENT_DATA:
-                                ib_verbs_fill_handshake_ack (buf, &priv->handshake.outgoing);
-                                priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_SENDING_ACK;
-                                break;
-
-                        case IB_VERBS_HANDSHAKE_SENDING_ACK:
-                                ret = __tcp_writev (this,
-                                                    &priv->handshake.outgoing.vector,
-                                                    priv->handshake.outgoing.count,
-                                                    &priv->handshake.outgoing.pending_vector,
-                                                    &priv->handshake.outgoing.pending_count);
-
-                                if (ret == -1) {
-                                        goto unlock;
-                                }
-
-                                if (ret > 0) {
-                                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                                "partial header read on NB "
-                                                "socket. continue later");
-                                        goto unlock;
-                                }
-            
-                                if (!ret) {
-                                        GF_FREE (priv->handshake.outgoing.buf);
-                                        priv->handshake.outgoing.buf = NULL;
-                                        priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_COMPLETE;
-                                }
-                                break;
-                        }
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (ret == -1) {
-                transport_disconnect (this);
-        } else {
-                ret = 0;
-        }
-
-        return ret;
-}
-
-static int
-ib_verbs_handshake_pollerr (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        int32_t ret = 0;
-        char need_unref = 0;
-
-        gf_log ("transport/ib-verbs", GF_LOG_DEBUG,
-                "%s: peer disconnected, cleaning up",
-                this->xl->name);
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                __ib_verbs_teardown (this);
-
-                if (priv->sock != -1) {
-                        event_unregister (this->xl->ctx->event_pool, 
-                                          priv->sock, priv->idx);
-                        need_unref = 1;
-
-                        if (close (priv->sock) != 0) {
-                                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                        "close () - error: %s",
-                                        strerror (errno));
-                                ret = -errno;
-                        }
-                        priv->tcp_connected = priv->connected = 0;
-                        priv->sock = -1;
-                }
-
-                if (priv->handshake.incoming.buf) {
-                        GF_FREE (priv->handshake.incoming.buf);
-                        priv->handshake.incoming.buf = NULL;
-                }
-
-                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_START;
-
-                if (priv->handshake.outgoing.buf) {
-                        GF_FREE (priv->handshake.outgoing.buf);
-                        priv->handshake.outgoing.buf = NULL;
-                }
-
-                priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_START;
-        }
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        xlator_notify (this->xl, GF_EVENT_POLLERR, this, NULL);
-
-        if (need_unref)
-                transport_unref (this);
-
-        return 0;
-}
-
-
-static int
-tcp_connect_finish (transport_t *this)
-{
-        ib_verbs_private_t *priv = this->private;
-        int error = 0, ret = 0;
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                ret = __tcp_connect_finish (priv->sock);
-
-                if (!ret) {
-                        this->myinfo.sockaddr_len = 
-                                sizeof (this->myinfo.sockaddr);
-                        ret = getsockname (priv->sock,
-                                           (struct sockaddr *)&this->myinfo.sockaddr, 
-                                           &this->myinfo.sockaddr_len);
-                        if (ret == -1) 
-                        {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "getsockname on new client-socket %d "
-                                        "failed (%s)", 
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                error = 1;
-                                goto unlock;
-                        }
-
-                        gf_ibverbs_get_transport_identifiers (this);
-                        priv->tcp_connected = 1;
-                }
-
-                if (ret == -1 && errno != EINPROGRESS) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "tcp connect to %s failed (%s)", 
-                                this->peerinfo.identifier, strerror (errno));
-                        error = 1;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        if (error) {
-                transport_disconnect (this);
-        }
-
-        return ret;
-}
-
-static int
-ib_verbs_event_handler (int fd, int idx, void *data,
-                        int poll_in, int poll_out, int poll_err)
-{
-        transport_t *this = data;
-        ib_verbs_private_t *priv = this->private;
-        ib_verbs_options_t *options = NULL;
-        int ret = 0;
-
-        if (!priv->tcp_connected) {
-                ret = tcp_connect_finish (this);
-                if (priv->tcp_connected) {
-                        options = &priv->options;
-
-                        priv->peer.send_count = options->send_count;
-                        priv->peer.recv_count = options->recv_count;
-                        priv->peer.send_size = options->send_size;
-                        priv->peer.recv_size = options->recv_size;
-
-                        if ((ret = ib_verbs_create_qp (this)) < 0) {
-                                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                        "%s: could not create QP",
-                                        this->xl->name);
-                                transport_disconnect (this);
-                        }
-                }
-        }
-
-        if (!ret && poll_out && priv->tcp_connected) {
-                ret = ib_verbs_handshake_pollout (this);
-        }
-
-        if (!ret && poll_in && priv->tcp_connected) {
-                if (priv->handshake.incoming.state == IB_VERBS_HANDSHAKE_COMPLETE) {
-                        gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                                "%s: pollin received on tcp socket (peer: %s) "
-                                "after handshake is complete",
-                                this->xl->name, this->peerinfo.identifier);
-                        ib_verbs_handshake_pollerr (this);
-                        return 0;
-                }
-                ret = ib_verbs_handshake_pollin (this);
-        }
-
-        if (ret < 0 || poll_err) {
-                ret = ib_verbs_handshake_pollerr (this);
-        }
-
-        return 0;
-}
-
-static int
-__tcp_nonblock (int fd)
-{
-        int flags = 0;
-        int ret = -1;
-
-        flags = fcntl (fd, F_GETFL);
-
-        if (flags != -1)
-                ret = fcntl (fd, F_SETFL, flags | O_NONBLOCK);
-
-        return ret;
-}
-
-static int32_t
-ib_verbs_connect (struct transport *this)
-{
-        dict_t *options = this->xl->options;
-  
-        ib_verbs_private_t *priv = this->private;
-  
-        int32_t ret = 0;
-        gf_boolean_t non_blocking = 1;
-        struct sockaddr_storage sockaddr;
-        socklen_t sockaddr_len = 0;
-
-        if (priv->connected) {
-                return 0;
-        }
-
-        if (dict_get (options, "non-blocking-io")) {
-                char *nb_connect = data_to_str (dict_get (this->xl->options,
-                                                          "non-blocking-io"));
-          
-                if (gf_string2boolean (nb_connect, &non_blocking) == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "'non-blocking-io' takes only boolean "
-                                "options, not taking any action");
-                        non_blocking = 1;
-                }
-        }
-
-        ret = gf_ibverbs_client_get_remote_sockaddr (this,
-                                                     (struct sockaddr *)&sockaddr,
-                                                     &sockaddr_len);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "cannot get remote address to connect");
-                return ret;
-        }
-
-        pthread_mutex_lock (&priv->write_mutex);
-        {
-                if (priv->sock != -1) {
-                        ret = 0;
-                        goto unlock;
-                }
-  
-                priv->sock = socket (((struct sockaddr *)&sockaddr)->sa_family,
-                                     SOCK_STREAM, 0);
-        
-                if (priv->sock == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "socket () - error: %s", strerror (errno));
-                        ret = -errno;
-                        goto unlock;
-                }
-
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "socket fd = %d", priv->sock);
-
-                memcpy (&this->peerinfo.sockaddr, &sockaddr, sockaddr_len);
-                this->peerinfo.sockaddr_len = sockaddr_len;
-
-                ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = 
-                        ((struct sockaddr *)&this->peerinfo.sockaddr)->sa_family;
-
-                if (non_blocking) 
-                {
-                        ret = __tcp_nonblock (priv->sock);
-        
-                        if (ret == -1)
-                        {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "could not set socket %d to non "
-                                        "blocking mode (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
-                        }
-                }
-
-                ret = gf_ibverbs_client_bind (this,
-                                              (struct sockaddr *)&this->myinfo.sockaddr,
-                                              &this->myinfo.sockaddr_len, priv->sock);
-                if (ret == -1)
-                {
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "client bind failed: %s", strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                ret = connect (priv->sock, 
-                               (struct sockaddr *)&this->peerinfo.sockaddr, 
-                               this->peerinfo.sockaddr_len);
-                if (ret == -1 && errno != EINPROGRESS)
-                {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "connection attempt failed (%s)", 
-                                strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                priv->tcp_connected = priv->connected = 0;
-
-                transport_ref (this);
-
-                priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_START;
-                priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_START;
-                        
-                priv->idx = event_register (this->xl->ctx->event_pool, 
-                                            priv->sock, ib_verbs_event_handler,
-                                            this, 1, 1); 
-        }
-unlock:
-        pthread_mutex_unlock (&priv->write_mutex);
-
-        return ret;
-}
-
-static int
-ib_verbs_server_event_handler (int fd, int idx, void *data,
-                               int poll_in, int poll_out, int poll_err)
-{
-        int32_t main_sock = -1;
-        transport_t *this, *trans = data;
-        ib_verbs_private_t *priv = NULL;
-        ib_verbs_private_t *trans_priv = (ib_verbs_private_t *) trans->private;
-        ib_verbs_options_t *options = NULL;
-
-        if (!poll_in)
-                return 0;
-
-        this = GF_CALLOC (1, sizeof (transport_t),
-                          gf_ibv_mt_transport_t);
-        if (!this)
-                return 0;
-
-        priv = GF_CALLOC (1, sizeof (ib_verbs_private_t),
-                          gf_ibv_mt_ib_verbs_private_t);
-        if (!priv) {
-                GF_FREE (this);
-                return 0;
-        }
-
-        this->private = priv;
-        /* Copy all the ib_verbs related values in priv, from trans_priv 
-           as other than QP, all the values remain same */
-        priv->device = trans_priv->device;
-        priv->options = trans_priv->options;
-        options = &priv->options;
-
-        this->ops = trans->ops;
-        this->xl = trans->xl;
-        this->init = trans->init;
-        this->fini = trans->fini;
-
-        memcpy (&this->myinfo.sockaddr, &trans->myinfo.sockaddr, 
-                trans->myinfo.sockaddr_len);
-        this->myinfo.sockaddr_len = trans->myinfo.sockaddr_len;
-
-        main_sock = (trans_priv)->sock;
-        this->peerinfo.sockaddr_len = sizeof (this->peerinfo.sockaddr);
-        priv->sock = accept (main_sock, 
-                             (struct sockaddr *)&this->peerinfo.sockaddr, 
-                             &this->peerinfo.sockaddr_len);
-        if (priv->sock == -1) {
-                gf_log ("ib-verbs/server", GF_LOG_ERROR,
-                        "accept() failed: %s",
-                        strerror (errno));
-                GF_FREE (this->private);
-                GF_FREE (this);
-                return -1;
-        }
-
-        priv->peer.trans = this;
-        transport_ref (this);
-
-        gf_ibverbs_get_transport_identifiers (this);
-
-        priv->tcp_connected = 1;
-        priv->handshake.incoming.state = IB_VERBS_HANDSHAKE_START;
-        priv->handshake.outgoing.state = IB_VERBS_HANDSHAKE_START;
-
-        priv->peer.send_count = options->send_count;
-        priv->peer.recv_count = options->recv_count;
-        priv->peer.send_size = options->send_size;
-        priv->peer.recv_size = options->recv_size;
-        INIT_LIST_HEAD (&priv->peer.ioq);
-
-        if (ib_verbs_create_qp (this) < 0) {
-                gf_log ("transport/ib-verbs", GF_LOG_ERROR,
-                        "%s: could not create QP",
-                        this->xl->name);
-                transport_disconnect (this);
-                return -1;
-        }
-
-        priv->idx = event_register (this->xl->ctx->event_pool, priv->sock,
-                                    ib_verbs_event_handler, this, 1, 1);
-
-        pthread_mutex_init (&priv->read_mutex, NULL);
-        pthread_mutex_init (&priv->write_mutex, NULL);
-        pthread_mutex_init (&priv->recv_mutex, NULL);
-        /*  pthread_cond_init (&priv->recv_cond, NULL); */
-
-        return 0;
-}
-
-static int32_t
-ib_verbs_listen (transport_t *this)
-{
-        struct sockaddr_storage sockaddr;
-        socklen_t sockaddr_len;
-        ib_verbs_private_t *priv = this->private;
-        int opt = 1, ret = 0;
-        char service[NI_MAXSERV], host[NI_MAXHOST];
-
-        memset (&sockaddr, 0, sizeof (sockaddr));
-        ret = gf_ibverbs_server_get_local_sockaddr (this,
-                                                    (struct sockaddr *)&sockaddr,
-                                                    &sockaddr_len);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "cannot find network address of server to bind to");
-                goto err;
-        }
-
-        priv->sock = socket (((struct sockaddr *)&sockaddr)->sa_family, 
-                             SOCK_STREAM, 0);
-        if (priv->sock == -1) {
-                gf_log ("ib-verbs/server", GF_LOG_CRITICAL,
-                        "init: failed to create socket, error: %s",
-                        strerror (errno));
-                GF_FREE (this->private);
-                ret = -1;
-                goto err;
-        }
-
-        memcpy (&this->myinfo.sockaddr, &sockaddr, sockaddr_len);
-        this->myinfo.sockaddr_len = sockaddr_len;
-
-        ret = getnameinfo ((struct sockaddr *)&this->myinfo.sockaddr, 
-                           this->myinfo.sockaddr_len,
-                           host, sizeof (host),
-                           service, sizeof (service),
-                           NI_NUMERICHOST);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "getnameinfo failed (%s)", gai_strerror (ret));
-                goto err;
-        }
-        sprintf (this->myinfo.identifier, "%s:%s", host, service);
- 
-        setsockopt (priv->sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof (opt));
-        if (bind (priv->sock,
-                  (struct sockaddr *)&sockaddr,
-                  sockaddr_len) != 0) {
-                ret = -1;
-                gf_log ("ib-verbs/server", GF_LOG_ERROR,
-                        "init: failed to bind to socket for %s (%s)",
-                        this->myinfo.identifier, strerror (errno));
-                goto err;
-        }
-
-        if (listen (priv->sock, 10) != 0) {
-                gf_log ("ib-verbs/server", GF_LOG_ERROR,
-                        "init: listen () failed on socket for %s (%s)",
-                        this->myinfo.identifier, strerror (errno));
-                ret = -1;
-                goto err;
-        }
-
-        /* Register the main socket */
-        priv->idx = event_register (this->xl->ctx->event_pool, priv->sock,
-                                    ib_verbs_server_event_handler, 
-                                    transport_ref (this), 1, 0);
-
-err:
-        return ret;
-}
-
-struct transport_ops tops = {
-        .receive = ib_verbs_receive,
-        .submit = ib_verbs_submit,
-        .connect = ib_verbs_connect,
-        .disconnect = ib_verbs_disconnect,
-        .listen = ib_verbs_listen,
-};
-
-int32_t
-init (transport_t *this)
-{
-        ib_verbs_private_t *priv = GF_CALLOC (1, sizeof (*priv),
-                                              gf_ibv_mt_ib_verbs_private_t);
-        this->private = priv;
-        priv->sock = -1;
-
-        if (ib_verbs_init (this)) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "Failed to initialize IB Device");
-                return -1;
-        }
-
-        return 0;
-}
-
-void  
-fini (struct transport *this)
-{
-        /* TODO: verify this function does graceful finish */
-        ib_verbs_private_t *priv = this->private;
-        this->private = NULL;
-
-        pthread_mutex_destroy (&priv->recv_mutex);
-        pthread_mutex_destroy (&priv->write_mutex);
-        pthread_mutex_destroy (&priv->read_mutex);
-        /*  pthread_cond_destroy (&priv->recv_cond); */
-
-        gf_log (this->xl->name, GF_LOG_TRACE,
-                "called fini on transport: %p",
-                this);
-        GF_FREE (priv);
-        return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_common_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-/* TODO: expand each option */
-struct volume_options options[] = {
-        { .key   = {"transport.ib-verbs.port",
-                    "ib-verbs-port"}, 
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 4,
-          .description = "check the option by 'ibv_devinfo'"
-        },
-        { .key   = {"transport.ib-verbs.mtu",
-                    "ib-verbs-mtu"}, 
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key   = {"transport.ib-verbs.device-name",
-                    "ib-verbs-device-name"}, 
-          .type  = GF_OPTION_TYPE_ANY,
-          .description = "check by 'ibv_devinfo'"
-        },
-        { .key   = {"transport.ib-verbs.work-request-send-count",
-                    "ib-verbs-work-request-send-count"}, 
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key   = {"transport.ib-verbs.work-request-recv-count",
-                    "ib-verbs-work-request-recv-count"}, 
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key   = {"remote-port", 
-                    "transport.remote-port",
-                    "transport.ib-verbs.remote-port"}, 
-          .type  = GF_OPTION_TYPE_INT 
-        },
-        { .key   = {"transport.ib-verbs.listen-port", "listen-port"}, 
-          .type  = GF_OPTION_TYPE_INT 
-        },
-        { .key   = {"transport.ib-verbs.connect-path", "connect-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.ib-verbs.bind-path", "bind-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.ib-verbs.listen-path", "listen-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.address-family",
-                    "address-family"}, 
-          .value = {"inet", "inet6", "inet/inet6", "inet6/inet",
-                    "unix", "inet-sdp" },
-          .type  = GF_OPTION_TYPE_STR 
-        },
-        { .key   = {"transport.socket.lowlat"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key = {NULL} }
-};
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.h b/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.h
deleted file mode 100644
index 3818609b0..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/ib-verbs.h
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _XPORT_IB_VERBS_H
-#define _XPORT_IB_VERBS_H
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef MAX_IOVEC
-#define MAX_IOVEC 16
-#endif /* MAX_IOVEC */
-
-#include "xlator.h"
-#include "event.h"
-#include "ib-verbs-mem-types.h"
-
-#include <stdio.h>
-#include <list.h>
-#include <arpa/inet.h>
-#include <infiniband/verbs.h>
-
-#define GF_DEFAULT_IBVERBS_LISTEN_PORT 6997
-
-/* options per transport end point */
-struct _ib_verbs_options {
-        int32_t port;
-        char *device_name;
-        enum ibv_mtu mtu;
-        int32_t  send_count;
-        int32_t  recv_count;
-        uint64_t recv_size;
-        uint64_t send_size;
-};
-typedef struct _ib_verbs_options ib_verbs_options_t;
-
-
-struct _ib_verbs_header {
-        char     colonO[3];
-        uint32_t size1;
-        uint32_t size2;
-        char     version;
-} __attribute__((packed));
-typedef struct _ib_verbs_header ib_verbs_header_t;
-
-struct _ib_verbs_ioq {
-        union {
-                struct list_head list;
-                struct {
-                        struct _ib_verbs_ioq    *next;
-                        struct _ib_verbs_ioq    *prev;
-                };
-        };
-        ib_verbs_header_t  header;
-        struct iovec       vector[MAX_IOVEC];
-        int                count;
-        char              *buf;
-        struct iobref     *iobref;
-};
-typedef struct _ib_verbs_ioq ib_verbs_ioq_t;
-
-/* represents one communication peer, two per transport_t */
-struct _ib_verbs_peer {
-        transport_t *trans;
-        struct ibv_qp *qp;
-
-        int32_t recv_count;
-        int32_t send_count;
-        int32_t recv_size;
-        int32_t send_size;
-
-        int32_t quota;
-        union {
-                struct list_head     ioq;
-                struct {
-                        ib_verbs_ioq_t        *ioq_next;
-                        ib_verbs_ioq_t        *ioq_prev;
-                };
-        };
-
-        /* QP attributes, needed to connect with remote QP */
-        int32_t local_lid;
-        int32_t local_psn;
-        int32_t local_qpn;
-        int32_t remote_lid;
-        int32_t remote_psn;
-        int32_t remote_qpn;
-};
-typedef struct _ib_verbs_peer ib_verbs_peer_t;
-
-
-struct _ib_verbs_post {
-        struct _ib_verbs_post *next, *prev;
-        struct ibv_mr *mr;
-        char *buf;
-        int32_t buf_size;
-        char aux;
-        int32_t reused;
-        pthread_barrier_t wait;
-};
-typedef struct _ib_verbs_post ib_verbs_post_t;
-
-
-struct _ib_verbs_queue {
-        ib_verbs_post_t active_posts, passive_posts;
-        int32_t active_count, passive_count;
-        pthread_mutex_t lock;
-};
-typedef struct _ib_verbs_queue ib_verbs_queue_t;
-
-
-struct _ib_verbs_qpreg {
-        pthread_mutex_t lock;
-        int32_t count;
-        struct _qpent {
-                struct _qpent *next, *prev;
-                int32_t qp_num;
-                ib_verbs_peer_t *peer;
-        } ents[42];
-};
-typedef struct _ib_verbs_qpreg ib_verbs_qpreg_t;
-
-/* context per device, stored in global glusterfs_ctx_t->ib */
-struct _ib_verbs_device {
-        struct _ib_verbs_device *next;
-        const char *device_name;
-        struct ibv_context *context;
-        int32_t port;
-        struct ibv_pd *pd;
-        struct ibv_srq *srq;
-        ib_verbs_qpreg_t qpreg;
-        struct ibv_comp_channel *send_chan, *recv_chan;
-        struct ibv_cq *send_cq, *recv_cq;
-        ib_verbs_queue_t sendq, recvq;
-        pthread_t send_thread, recv_thread;
-};
-typedef struct _ib_verbs_device ib_verbs_device_t;
-
-typedef enum {
-        IB_VERBS_HANDSHAKE_START = 0,
-        IB_VERBS_HANDSHAKE_SENDING_DATA,
-        IB_VERBS_HANDSHAKE_RECEIVING_DATA,
-        IB_VERBS_HANDSHAKE_SENT_DATA,
-        IB_VERBS_HANDSHAKE_RECEIVED_DATA,
-        IB_VERBS_HANDSHAKE_SENDING_ACK,
-        IB_VERBS_HANDSHAKE_RECEIVING_ACK,
-        IB_VERBS_HANDSHAKE_RECEIVED_ACK,
-        IB_VERBS_HANDSHAKE_COMPLETE,
-} ib_verbs_handshake_state_t;
-
-struct ib_verbs_nbio {
-        int state;
-        char *buf;
-        int count;
-        struct iovec vector;
-        struct iovec *pending_vector;
-        int pending_count;
-};
-
-
-struct _ib_verbs_private {
-        int32_t sock;
-        int32_t idx;
-        unsigned char connected;
-        unsigned char tcp_connected;
-        unsigned char ib_connected;
-        in_addr_t addr;
-        unsigned short port;
-
-        /* IB Verbs Driver specific variables, pointers */
-        ib_verbs_peer_t peer;
-        ib_verbs_device_t *device;
-        ib_verbs_options_t options;
-
-        /* Used by trans->op->receive */
-        char *data_ptr;
-        int32_t data_offset;
-        int32_t data_len;
-
-        /* Mutex */
-        pthread_mutex_t read_mutex;
-        pthread_mutex_t write_mutex;
-        pthread_barrier_t handshake_barrier;
-        char handshake_ret;
-
-        pthread_mutex_t recv_mutex;
-        pthread_cond_t  recv_cond;
-
-        /* used during ib_verbs_handshake */
-        struct {
-                struct ib_verbs_nbio incoming;
-                struct ib_verbs_nbio outgoing;
-                int               state;
-                ib_verbs_header_t header;
-                char *buf;
-                size_t size;
-        } handshake;
-};
-typedef struct _ib_verbs_private ib_verbs_private_t;
-
-#endif /* _XPORT_IB_VERBS_H */
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/name.c b/xlators/protocol/legacy/transport/ib-verbs/src/name.c
deleted file mode 100644
index 511a33a22..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/name.c
+++ /dev/null
@@ -1,712 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <errno.h>
-#include <netdb.h>
-#include <string.h>
-
-#ifdef CLIENT_PORT_CEILING
-#undef CLIENT_PORT_CEILING
-#endif
-
-#define CLIENT_PORT_CEILING 1024
-
-#ifndef AF_INET_SDP
-#define AF_INET_SDP 27
-#endif
-
-#include "transport.h"
-#include "ib-verbs.h"
-
-int32_t
-gf_resolve_ip6 (const char *hostname, 
-                uint16_t port, 
-                int family, 
-                void **dnscache, 
-                struct addrinfo **addr_info);
-
-static int32_t
-af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr, 
-                                 socklen_t sockaddr_len, int ceiling)
-{
-        int32_t ret = -1;
-        /*  struct sockaddr_in sin = {0, }; */
-        uint16_t port = ceiling - 1;
-
-        while (port)
-        {
-                switch (sockaddr->sa_family)
-                {
-                case AF_INET6:
-                        ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
-                        break;
-
-                case AF_INET_SDP:
-                case AF_INET:
-                        ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
-                        break;
-                }
-
-                ret = bind (fd, sockaddr, sockaddr_len);
-
-                if (ret == 0)
-                        break;
-
-                if (ret == -1 && errno == EACCES)
-                        break;
-
-                port--;
-        }
-
-        return ret;
-}
-
-static int32_t
-af_unix_client_bind (transport_t *this, 
-                     struct sockaddr *sockaddr, 
-                     socklen_t sockaddr_len, 
-                     int sock)
-{
-        data_t *path_data = NULL;
-        struct sockaddr_un *addr = NULL;
-        int32_t ret = -1;
-
-        path_data = dict_get (this->xl->options, 
-                              "transport.ib-verbs.bind-path");
-        if (path_data) {
-                char *path = data_to_str (path_data);
-                if (!path || strlen (path) > UNIX_PATH_MAX) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "transport.ib-verbs.bind-path not specfied "
-                                "for unix socket, letting connect to assign "
-                                "default value");
-                        goto err;
-                }
-
-                addr = (struct sockaddr_un *) sockaddr;
-                strcpy (addr->sun_path, path);
-                ret = bind (sock, (struct sockaddr *)addr, sockaddr_len);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "cannot bind to unix-domain socket %d (%s)", 
-                                sock, strerror (errno));
-                        goto err;
-                }
-        }
-
-err:
-        return ret;
-}
-
-static int32_t
-client_fill_address_family (transport_t *this, struct sockaddr *sockaddr)
-{
-        data_t *address_family_data = NULL;
-
-        address_family_data = dict_get (this->xl->options, 
-                                        "transport.address-family");
-        if (!address_family_data) {
-                data_t *remote_host_data = NULL, *connect_path_data = NULL;
-                remote_host_data = dict_get (this->xl->options, "remote-host");
-                connect_path_data = dict_get (this->xl->options, 
-                                              "transport.ib-verbs.connect-path");
-
-                if (!(remote_host_data || connect_path_data) || 
-                    (remote_host_data && connect_path_data)) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "address-family not specified and not able to "
-                                "determine the same from other options "
-                                "(remote-host:%s and connect-path:%s)", 
-                                data_to_str (remote_host_data), 
-                                data_to_str (connect_path_data));
-                        return -1;
-                } 
-
-                if (remote_host_data) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be inet/inet6");
-                        sockaddr->sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be unix");
-                        sockaddr->sa_family = AF_UNIX;
-                }
-
-        } else {
-                char *address_family = data_to_str (address_family_data);
-                if (!strcasecmp (address_family, "unix")) {
-                        sockaddr->sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet")) {
-                        sockaddr->sa_family = AF_INET;
-                } else if (!strcasecmp (address_family, "inet6")) {
-                        sockaddr->sa_family = AF_INET6;
-                } else if (!strcasecmp (address_family, "inet-sdp")) {
-                        sockaddr->sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        sockaddr->sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unknown address-family (%s) specified", 
-                                address_family);
-                        return -1;
-                }
-        }
-
-        return 0;
-}
-
-static int32_t
-af_inet_client_get_remote_sockaddr (transport_t *this, 
-                                    struct sockaddr *sockaddr, 
-                                    socklen_t *sockaddr_len)
-{
-        dict_t *options = this->xl->options;
-        data_t *remote_host_data = NULL;
-        data_t *remote_port_data = NULL;
-        char *remote_host = NULL;
-        uint16_t remote_port = 0;
-        struct addrinfo *addr_info = NULL;
-        int32_t ret = 0;
-
-        remote_host_data = dict_get (options, "remote-host");
-        if (remote_host_data == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-host missing in volume %s", 
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        remote_host = data_to_str (remote_host_data);
-        if (remote_host == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-host has data NULL in volume %s", 
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        remote_port_data = dict_get (options, "remote-port");
-        if (remote_port_data == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "option remote-port missing in volume %s. "
-                        "Defaulting to %d",
-                        this->xl->name, GF_DEFAULT_IBVERBS_LISTEN_PORT);
-
-                remote_port = GF_DEFAULT_IBVERBS_LISTEN_PORT;
-        }
-        else
-        {
-                remote_port = data_to_uint16 (remote_port_data);
-        }
-
-        if (remote_port == (uint16_t)-1)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-port has invalid port in volume %s",
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        /* TODO: gf_resolve is a blocking call. kick in some
-           non blocking dns techniques */
-        ret = gf_resolve_ip6 (remote_host, remote_port,
-                              sockaddr->sa_family, 
-                              &this->dnscache, &addr_info);
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "DNS resolution failed on host %s", remote_host);
-                goto err;
-        }
-
-        memcpy (sockaddr, addr_info->ai_addr, addr_info->ai_addrlen);
-        *sockaddr_len = addr_info->ai_addrlen;
-
-err:
-        return ret;
-}
-
-static int32_t
-af_unix_client_get_remote_sockaddr (transport_t *this, 
-                                    struct sockaddr *sockaddr, 
-                                    socklen_t *sockaddr_len)
-{
-        struct sockaddr_un *sockaddr_un = NULL;
-        char *connect_path = NULL;
-        data_t *connect_path_data = NULL;
-        int32_t ret = 0;
-
-        connect_path_data = dict_get (this->xl->options, 
-                                      "transport.ib-verbs.connect-path");
-        if (!connect_path_data) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option transport.ib-verbs.connect-path not "
-                        "specified for address-family unix");
-                ret = -1;
-                goto err;
-        }
-
-        connect_path = data_to_str (connect_path_data);
-        if (!connect_path) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "connect-path is null-string");
-                ret = -1;
-                goto err;
-        }
-
-        if (strlen (connect_path) > UNIX_PATH_MAX) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "connect-path value length %"GF_PRI_SIZET" > "
-                        "%d octets", strlen (connect_path), UNIX_PATH_MAX);
-                ret = -1;
-                goto err;
-        }
-
-        gf_log (this->xl->name,
-                GF_LOG_DEBUG,
-                "using connect-path %s", connect_path);
-        sockaddr_un = (struct sockaddr_un *)sockaddr;
-        strcpy (sockaddr_un->sun_path, connect_path);
-        *sockaddr_len = sizeof (struct sockaddr_un);
-
-err:
-        return ret;
-}
-
-static int32_t
-af_unix_server_get_local_sockaddr (transport_t *this,
-                                   struct sockaddr *addr,
-                                   socklen_t *addr_len)
-{
-        data_t *listen_path_data = NULL;
-        char *listen_path = NULL;
-        int32_t ret = 0;
-        struct sockaddr_un *sunaddr = (struct sockaddr_un *)addr;
-
-
-        listen_path_data = dict_get (this->xl->options, 
-                                     "transport.ib-verbs.listen-path");
-        if (!listen_path_data) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "missing option listen-path");
-                ret = -1;
-                goto err;
-        }
-
-        listen_path = data_to_str (listen_path_data);
-
-#ifndef UNIX_PATH_MAX
-#define UNIX_PATH_MAX 108
-#endif
-
-        if (strlen (listen_path) > UNIX_PATH_MAX) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option listen-path has value length %"GF_PRI_SIZET" > %d",
-                        strlen (listen_path), UNIX_PATH_MAX);
-                ret = -1;
-                goto err;
-        }
-
-        sunaddr->sun_family = AF_UNIX;
-        strcpy (sunaddr->sun_path, listen_path);
-        *addr_len = sizeof (struct sockaddr_un);
-
-err:
-        return ret;
-}
-
-static int32_t 
-af_inet_server_get_local_sockaddr (transport_t *this, 
-                                   struct sockaddr *addr, 
-                                   socklen_t *addr_len)
-{
-        struct addrinfo hints, *res = 0;
-        data_t *listen_port_data = NULL, *listen_host_data = NULL;
-        uint16_t listen_port = -1;
-        char service[NI_MAXSERV], *listen_host = NULL;
-        dict_t *options = NULL;
-        int32_t ret = 0;
-
-        options = this->xl->options;
-
-        listen_port_data = dict_get (options, "transport.ib-verbs.listen-port");
-        listen_host_data = dict_get (options, "transport.ib-verbs.bind-address");
-
-        if (listen_port_data)
-        {
-                listen_port = data_to_uint16 (listen_port_data);
-        } else {
-		if (addr->sa_family == AF_INET6) {
-			struct sockaddr_in6 *in = (struct sockaddr_in6 *) addr;
-			in->sin6_addr = in6addr_any;
-			in->sin6_port = htons(listen_port);
-			*addr_len = sizeof(struct sockaddr_in6);
-                        goto out;
-		} else if (addr->sa_family == AF_INET) {
-			struct sockaddr_in *in = (struct sockaddr_in *) addr;
-			in->sin_addr.s_addr = htonl(INADDR_ANY);
-			in->sin_port = htons(listen_port);
-			*addr_len = sizeof(struct sockaddr_in);
-                        goto out;
-		}
-	}
-
-        if (listen_port == (uint16_t) -1)
-                listen_port = GF_DEFAULT_IBVERBS_LISTEN_PORT;
-
-
-        if (listen_host_data)
-        {
-                listen_host = data_to_str (listen_host_data);
-        }
-
-        memset (service, 0, sizeof (service));
-        sprintf (service, "%d", listen_port);
-
-        memset (&hints, 0, sizeof (hints));
-        hints.ai_family = addr->sa_family;
-        hints.ai_socktype = SOCK_STREAM;
-        hints.ai_flags    = AI_ADDRCONFIG | AI_PASSIVE;
-
-        ret = getaddrinfo(listen_host, service, &hints, &res);
-        if (ret != 0) {
-                gf_log (this->xl->name,
-                        GF_LOG_ERROR,
-                        "getaddrinfo failed for host %s, service %s (%s)", 
-                        listen_host, service, gai_strerror (ret));
-                ret = -1;
-                goto out;
-        }
-
-        memcpy (addr, res->ai_addr, res->ai_addrlen);
-        *addr_len = res->ai_addrlen;
-
-        freeaddrinfo (res);
-
-out:
-        return ret;
-}
-
-int32_t
-gf_ibverbs_client_bind (transport_t *this,
-                        struct sockaddr *sockaddr,
-                        socklen_t *sockaddr_len,
-                        int sock)
-{
-        int ret = 0;
-
-        *sockaddr_len = sizeof (struct sockaddr_in6);
-        switch (sockaddr->sa_family)
-        {
-        case AF_INET_SDP:
-        case AF_INET:
-                *sockaddr_len = sizeof (struct sockaddr_in);
-
-        case AF_INET6:
-                ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr, 
-                                                       *sockaddr_len, 
-                                                       CLIENT_PORT_CEILING);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "cannot bind inet socket (%d) to port "
-                                "less than %d (%s)", 
-                                sock, CLIENT_PORT_CEILING, strerror (errno));
-                        ret = 0;
-                }
-                break;
-
-        case AF_UNIX:
-                *sockaddr_len = sizeof (struct sockaddr_un);
-                ret = af_unix_client_bind (this, (struct sockaddr *)sockaddr, 
-                                           *sockaddr_len, sock);
-                break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address family %d", sockaddr->sa_family);
-                ret = -1;
-                break;
-        }
-
-        return ret;
-}
-
-int32_t
-gf_ibverbs_client_get_remote_sockaddr (transport_t *this,
-                                       struct sockaddr *sockaddr,
-                                       socklen_t *sockaddr_len)
-{
-        int32_t ret = 0;
-        char is_inet_sdp = 0;
-
-        ret = client_fill_address_family (this, sockaddr);
-        if (ret) {
-                ret = -1;
-                goto err;
-        }
- 
-        switch (sockaddr->sa_family)
-        {
-        case AF_INET_SDP:
-                sockaddr->sa_family = AF_INET;
-                is_inet_sdp = 1;
-
-        case AF_INET:
-        case AF_INET6:
-        case AF_UNSPEC:
-                ret = af_inet_client_get_remote_sockaddr (this, 
-                                                          sockaddr, 
-                                                          sockaddr_len);
-
-                if (is_inet_sdp) {
-                        sockaddr->sa_family = AF_INET_SDP;
-                }
-
-                break;
-
-        case AF_UNIX:
-                ret = af_unix_client_get_remote_sockaddr (this, 
-                                                          sockaddr, 
-                                                          sockaddr_len);
-                break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address-family %d", sockaddr->sa_family);
-                ret = -1;
-        }
-  
-err:
-        return ret;
-}
-
-int32_t
-gf_ibverbs_server_get_local_sockaddr (transport_t *this,
-                                      struct sockaddr *addr,
-                                      socklen_t *addr_len)
-{
-        data_t *address_family_data = NULL;
-        int32_t ret = 0;
-        char is_inet_sdp = 0;
-
-        address_family_data = dict_get (this->xl->options, 
-                                        "transport.address-family");
-        if (address_family_data) {
-                char *address_family = NULL;
-                address_family = data_to_str (address_family_data);
-
-                if (!strcasecmp (address_family, "inet")) {
-                        addr->sa_family = AF_INET;
-                } else if (!strcasecmp (address_family, "inet6")) {
-                        addr->sa_family = AF_INET6;
-                } else if (!strcasecmp (address_family, "inet-sdp")) {
-                        addr->sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "unix")) {
-                        addr->sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        addr->sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unknown address family (%s) specified", 
-                                address_family);
-                        ret = -1;
-                        goto err;
-                }
-        } else {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "option address-family not specified, defaulting "
-                        "to inet/inet6");
-                addr->sa_family = AF_UNSPEC;
-        }
-
-        switch (addr->sa_family)
-        {
-        case AF_INET_SDP:
-                is_inet_sdp = 1;
-                addr->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        case AF_UNSPEC:
-                ret = af_inet_server_get_local_sockaddr (this, addr, addr_len);
-                if (is_inet_sdp && !ret) {
-                        addr->sa_family = AF_INET_SDP;
-                }
-                break;
-
-        case AF_UNIX:
-                ret = af_unix_server_get_local_sockaddr (this, addr, addr_len);
-                break;
-        }
-
-err:
-        return ret;
-}
-
-int32_t 
-fill_inet6_inet_identifiers (transport_t *this, struct sockaddr_storage *addr, 
-                             int32_t addr_len, char *identifier)
-{
-        int32_t ret = 0, tmpaddr_len = 0;
-        char service[NI_MAXSERV], host[NI_MAXHOST];
-        struct sockaddr_storage tmpaddr;
-
-        memset (&tmpaddr, 0, sizeof (tmpaddr));
-        tmpaddr = *addr;
-        tmpaddr_len = addr_len;
-
-        if (((struct sockaddr *) &tmpaddr)->sa_family == AF_INET6) {
-                int32_t one_to_four, four_to_eight, twelve_to_sixteen;
-                int16_t eight_to_ten, ten_to_twelve;
-    
-                one_to_four = four_to_eight = twelve_to_sixteen = 0;
-                eight_to_ten = ten_to_twelve = 0;
-    
-                one_to_four = ((struct sockaddr_in6 *) 
-                               &tmpaddr)->sin6_addr.s6_addr32[0];
-                four_to_eight = ((struct sockaddr_in6 *) 
-                                 &tmpaddr)->sin6_addr.s6_addr32[1];
-#ifdef GF_SOLARIS_HOST_OS
-                eight_to_ten = S6_ADDR16(((struct sockaddr_in6 *) 
-                                          &tmpaddr)->sin6_addr)[4];
-#else
-                eight_to_ten = ((struct sockaddr_in6 *) 
-                                &tmpaddr)->sin6_addr.s6_addr16[4];
-#endif
-
-#ifdef GF_SOLARIS_HOST_OS
-                ten_to_twelve = S6_ADDR16(((struct sockaddr_in6 *) 
-                                           &tmpaddr)->sin6_addr)[5];
-#else
-                ten_to_twelve = ((struct sockaddr_in6 *) 
-                                 &tmpaddr)->sin6_addr.s6_addr16[5];
-#endif
-                twelve_to_sixteen = ((struct sockaddr_in6 *) 
-                                     &tmpaddr)->sin6_addr.s6_addr32[3];
-
-                /* ipv4 mapped ipv6 address has
-                   bits 0-80: 0
-                   bits 80-96: 0xffff
-                   bits 96-128: ipv4 address 
-                */
- 
-                if (one_to_four == 0 &&
-                    four_to_eight == 0 &&
-                    eight_to_ten == 0 &&
-                    ten_to_twelve == -1) {
-                        struct sockaddr_in *in_ptr = (struct sockaddr_in *)&tmpaddr;
-                        memset (&tmpaddr, 0, sizeof (tmpaddr));
-      
-                        in_ptr->sin_family = AF_INET;
-                        in_ptr->sin_port = ((struct sockaddr_in6 *)addr)->sin6_port;
-                        in_ptr->sin_addr.s_addr = twelve_to_sixteen;
-                        tmpaddr_len = sizeof (*in_ptr);
-                }
-        }
-
-        ret = getnameinfo ((struct sockaddr *) &tmpaddr,
-                           tmpaddr_len,
-                           host, sizeof (host),
-                           service, sizeof (service),
-                           NI_NUMERICHOST | NI_NUMERICSERV);
-        if (ret != 0) {
-                gf_log (this->xl->name,
-                        GF_LOG_ERROR,
-                        "getnameinfo failed (%s)", gai_strerror (ret));
-        }
-
-        sprintf (identifier, "%s:%s", host, service);
-
-        return ret;
-}
-
-int32_t
-gf_ibverbs_get_transport_identifiers (transport_t *this)
-{
-        int32_t ret = 0;
-        char is_inet_sdp = 0;
-
-        switch (((struct sockaddr *) &this->myinfo.sockaddr)->sa_family)
-        {
-        case AF_INET_SDP:
-                is_inet_sdp = 1;
-                ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        {
-                ret = fill_inet6_inet_identifiers (this, 
-                                                   &this->myinfo.sockaddr, 
-                                                   this->myinfo.sockaddr_len,
-                                                   this->myinfo.identifier);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "can't fill inet/inet6 identifier for server");
-                        goto err;
-                }
-
-                ret = fill_inet6_inet_identifiers (this,
-                                                   &this->peerinfo.sockaddr,
-                                                   this->peerinfo.sockaddr_len,
-                                                   this->peerinfo.identifier);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "can't fill inet/inet6 identifier for client");
-                        goto err;
-                }
-
-                if (is_inet_sdp) {
-                        ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET_SDP;
-                }
-        }
-        break;
-
-        case AF_UNIX:
-        {
-                struct sockaddr_un *sunaddr = NULL;
-
-                sunaddr = (struct sockaddr_un *) &this->myinfo.sockaddr;
-                strcpy (this->myinfo.identifier, sunaddr->sun_path);
-
-                sunaddr = (struct sockaddr_un *) &this->peerinfo.sockaddr;
-                strcpy (this->peerinfo.identifier, sunaddr->sun_path);
-        }
-        break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR, 
-                        "unknown address family (%d)",
-                        ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family);
-                ret = -1;
-                break;
-        }
-
-err:
-        return ret;
-}
diff --git a/xlators/protocol/legacy/transport/ib-verbs/src/name.h b/xlators/protocol/legacy/transport/ib-verbs/src/name.h
deleted file mode 100644
index 366077fdd..000000000
--- a/xlators/protocol/legacy/transport/ib-verbs/src/name.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _IB_VERBS_NAME_H
-#define _IB_VERBS_NAME_H
-
-#include <sys/socket.h>
-#include <sys/un.h>
-
-#include "compat.h"
-
-int32_t
-gf_ibverbs_client_bind (transport_t *this,
-                        struct sockaddr *sockaddr,
-                        socklen_t *sockaddr_len,
-                        int sock);
-
-int32_t
-gf_ibverbs_client_get_remote_sockaddr (transport_t *this,
-                                    struct sockaddr *sockaddr,
-                                    socklen_t *sockaddr_len);
-
-int32_t
-gf_ibverbs_server_get_local_sockaddr (transport_t *this,
-                                   struct sockaddr *addr,
-                                   socklen_t *addr_len);
-
-int32_t
-gf_ibverbs_get_transport_identifiers (transport_t *this);
-
-#endif /* _IB_VERBS_NAME_H */
diff --git a/xlators/protocol/legacy/transport/socket/Makefile.am b/xlators/protocol/legacy/transport/socket/Makefile.am
deleted file mode 100644
index f963effea..000000000
--- a/xlators/protocol/legacy/transport/socket/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = src
-\ No newline at end of file
diff --git a/xlators/protocol/legacy/transport/socket/src/Makefile.am b/xlators/protocol/legacy/transport/socket/src/Makefile.am
deleted file mode 100644
index 5952e18e9..000000000
--- a/xlators/protocol/legacy/transport/socket/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-# TODO : change to proper transport dir
-
-transport_LTLIBRARIES = socket.la
-transportdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/transport
-
-socket_la_LDFLAGS = -module -avoidversion
-
-socket_la_SOURCES = socket.c name.c
-socket_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	$(top_builddir)/xlators/protocol/legacy/lib/src/libgfproto.la
-
-noinst_HEADERS = socket.h name.h socket-mem-types.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
-	-I$(top_srcdir)/xlators/protocol/legacy/transport/socket/src   \
-	-I$(top_srcdir)/xlators/protocol/legacy/lib/src
-
-CLEANFILES = *~
diff --git a/xlators/protocol/legacy/transport/socket/src/name.c b/xlators/protocol/legacy/transport/socket/src/name.c
deleted file mode 100644
index 644a0bc5e..000000000
--- a/xlators/protocol/legacy/transport/socket/src/name.c
+++ /dev/null
@@ -1,740 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <errno.h>
-#include <netdb.h>
-#include <string.h>
-
-#ifdef CLIENT_PORT_CEILING
-#undef CLIENT_PORT_CEILING
-#endif
-
-#define CLIENT_PORT_CEILING 1024
-
-#ifndef AF_INET_SDP
-#define AF_INET_SDP 27
-#endif
-
-static int gf_name_addr_enotspec_log;
-
-#include "transport.h"
-#include "socket.h"
-
-int32_t
-gf_resolve_ip6 (const char *hostname, 
-                uint16_t port, 
-                int family, 
-                void **dnscache, 
-                struct addrinfo **addr_info);
-
-static int32_t
-af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr, 
-                                 socklen_t sockaddr_len, int ceiling)
-{
-        int32_t ret = -1;
-        /*  struct sockaddr_in sin = {0, }; */
-        uint16_t port = ceiling - 1;
-
-        while (port)
-        {
-                switch (sockaddr->sa_family)
-                {
-                case AF_INET6:
-                        ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
-                        break;
-
-                case AF_INET_SDP:
-                case AF_INET:
-                        ((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
-                        break;
-                }
-
-                ret = bind (fd, sockaddr, sockaddr_len);
-
-                if (ret == 0)
-                        break;
-
-                if (ret == -1 && errno == EACCES)
-                        break;
-
-                port--;
-        }
-
-        return ret;
-}
-
-static int32_t
-af_unix_client_bind (transport_t *this, 
-                     struct sockaddr *sockaddr, 
-                     socklen_t sockaddr_len, 
-                     int sock)
-{
-        data_t *path_data = NULL;
-        struct sockaddr_un *addr = NULL;
-        int32_t ret = 0;
-
-        path_data = dict_get (this->xl->options, "transport.socket.bind-path");
-        if (path_data) {
-                char *path = data_to_str (path_data);
-                if (!path || strlen (path) > UNIX_PATH_MAX) {
-                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                "bind-path not specfied for unix socket, "
-                                "letting connect to assign default value");
-                        goto err;
-                }
-
-                addr = (struct sockaddr_un *) sockaddr;
-                strcpy (addr->sun_path, path);
-                ret = bind (sock, (struct sockaddr *)addr, sockaddr_len);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "cannot bind to unix-domain socket %d (%s)", 
-                                sock, strerror (errno));
-                        goto err;
-                }
-        } else {
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "bind-path not specfied for unix socket, "
-                        "letting connect to assign default value");
-        }
-
-err:
-        return ret;
-}
-
-static int32_t
-client_fill_address_family (transport_t *this, sa_family_t *sa_family)
-{
-        data_t  *address_family_data = NULL;
-        int32_t  ret                 = -1;
-
-        if (sa_family == NULL) {
-                goto out;
-        }
-
-        address_family_data = dict_get (this->xl->options, 
-                                        "transport.address-family");
-        if (!address_family_data) {
-                data_t *remote_host_data = NULL, *connect_path_data = NULL;
-                remote_host_data = dict_get (this->xl->options, "remote-host");
-                connect_path_data = dict_get (this->xl->options, 
-                                              "transport.socket.connect-path");
-
-                if (!(remote_host_data || connect_path_data) || 
-                    (remote_host_data && connect_path_data)) {
-                        GF_LOG_OCCASIONALLY (gf_name_addr_enotspec_log, 
-                                             this->xl->name, GF_LOG_ERROR,
-                                             "transport.address-family not specified and "
-                                             "not able to determine the "
-                                             "same from other options (remote-host:%s and "
-                                             "transport.unix.connect-path:%s)", 
-                                             data_to_str (remote_host_data), 
-                                             data_to_str (connect_path_data));
-                        goto out;
-                } 
-
-                if (remote_host_data) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be inet/inet6");
-                        *sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "address-family not specified, guessing it "
-                                "to be unix");
-                        *sa_family = AF_UNIX;
-                }
-
-        } else {
-                char *address_family = data_to_str (address_family_data);
-                if (!strcasecmp (address_family, "unix")) {
-                        *sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet")) {
-                        *sa_family = AF_INET;
-                } else if (!strcasecmp (address_family, "inet6")) {
-                        *sa_family = AF_INET6;
-                } else if (!strcasecmp (address_family, "inet-sdp")) {
-                        *sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        *sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unknown address-family (%s) specified", 
-                                address_family);
-                        goto out;
-                }
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static int32_t
-af_inet_client_get_remote_sockaddr (transport_t *this, 
-                                    struct sockaddr *sockaddr, 
-                                    socklen_t *sockaddr_len)
-{
-        dict_t *options = this->xl->options;
-        data_t *remote_host_data = NULL;
-        data_t *remote_port_data = NULL;
-        char *remote_host = NULL;
-        uint16_t remote_port = 0;
-        struct addrinfo *addr_info = NULL;
-        int32_t ret = 0;
-
-        remote_host_data = dict_get (options, "remote-host");
-        if (remote_host_data == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-host missing in volume %s", this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        remote_host = data_to_str (remote_host_data);
-        if (remote_host == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-host has data NULL in volume %s", this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        remote_port_data = dict_get (options, "remote-port");
-        if (remote_port_data == NULL)
-        {
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "option remote-port missing in volume %s. Defaulting to %d",
-                        this->xl->name, GF_DEFAULT_SOCKET_LISTEN_PORT);
-
-                remote_port = GF_DEFAULT_SOCKET_LISTEN_PORT;
-        }
-        else
-        {
-                remote_port = data_to_uint16 (remote_port_data);
-        }
-
-        if (remote_port == (uint16_t)-1)
-        {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option remote-port has invalid port in volume %s",
-                        this->xl->name);
-                ret = -1;
-                goto err;
-        }
-
-        /* TODO: gf_resolve is a blocking call. kick in some
-           non blocking dns techniques */
-        ret = gf_resolve_ip6 (remote_host, remote_port,
-                              sockaddr->sa_family, &this->dnscache, &addr_info);
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "DNS resolution failed on host %s", remote_host);
-                goto err;
-        }
-
-        memcpy (sockaddr, addr_info->ai_addr, addr_info->ai_addrlen);
-        *sockaddr_len = addr_info->ai_addrlen;
-
-err:
-        return ret;
-}
-
-static int32_t
-af_unix_client_get_remote_sockaddr (transport_t *this, 
-                                    struct sockaddr *sockaddr, 
-                                    socklen_t *sockaddr_len)
-{
-        struct sockaddr_un *sockaddr_un = NULL;
-        char *connect_path = NULL;
-        data_t *connect_path_data = NULL;
-        int32_t ret = 0;
-
-        connect_path_data = dict_get (this->xl->options, 
-                                      "transport.socket.connect-path");
-        if (!connect_path_data) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option transport.unix.connect-path not specified for "
-                        "address-family unix");
-                ret = -1;
-                goto err;
-        }
-
-        connect_path = data_to_str (connect_path_data);
-        if (!connect_path) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "transport.unix.connect-path is null-string");
-                ret = -1;
-                goto err;
-        }
-
-        if (strlen (connect_path) > UNIX_PATH_MAX) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "connect-path value length %"GF_PRI_SIZET" > %d octets", 
-                        strlen (connect_path), UNIX_PATH_MAX);
-                ret = -1;
-                goto err;
-        }
-
-        gf_log (this->xl->name, GF_LOG_TRACE,
-                "using connect-path %s", connect_path);
-        sockaddr_un = (struct sockaddr_un *)sockaddr;
-        strcpy (sockaddr_un->sun_path, connect_path);
-        *sockaddr_len = sizeof (struct sockaddr_un);
-
-err:
-        return ret;
-}
-
-static int32_t
-af_unix_server_get_local_sockaddr (transport_t *this,
-                                   struct sockaddr *addr,
-                                   socklen_t *addr_len)
-{
-        data_t *listen_path_data = NULL;
-        char *listen_path = NULL;
-        int32_t ret = 0;
-        struct sockaddr_un *sunaddr = (struct sockaddr_un *)addr;
-
-
-        listen_path_data = dict_get (this->xl->options, 
-                                     "transport.socket.listen-path");
-        if (!listen_path_data) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "missing option transport.socket.listen-path");
-                ret = -1;
-                goto err;
-        }
-
-        listen_path = data_to_str (listen_path_data);
-
-#ifndef UNIX_PATH_MAX
-#define UNIX_PATH_MAX 108
-#endif
-
-        if (strlen (listen_path) > UNIX_PATH_MAX) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "option transport.unix.listen-path has value length "
-                        "%"GF_PRI_SIZET" > %d",
-                        strlen (listen_path), UNIX_PATH_MAX);
-                ret = -1;
-                goto err;
-        }
-
-        sunaddr->sun_family = AF_UNIX;
-        strcpy (sunaddr->sun_path, listen_path);
-        *addr_len = sizeof (struct sockaddr_un);
-
-err:
-        return ret;
-}
-
-static int32_t
-af_inet_server_get_local_sockaddr (transport_t *this,
-                                   struct sockaddr *addr,
-                                   socklen_t *addr_len)
-{
-        struct addrinfo hints, *res = 0;
-        data_t *listen_port_data = NULL, *listen_host_data = NULL;
-        uint16_t listen_port = -1;
-        char service[NI_MAXSERV], *listen_host = NULL;
-        dict_t *options = NULL;
-        int32_t ret = 0;
-
-        options = this->xl->options;
-
-        listen_port_data = dict_get (options, "transport.socket.listen-port");
-        listen_host_data = dict_get (options, "transport.socket.bind-address");
-
-        if (listen_port_data)
-        {
-                listen_port = data_to_uint16 (listen_port_data);
-        }
-
-        if (listen_port == (uint16_t) -1)
-                listen_port = GF_DEFAULT_SOCKET_LISTEN_PORT;
-
-
-        if (listen_host_data)
-        {
-                listen_host = data_to_str (listen_host_data);
-        } else {
-		if (addr->sa_family == AF_INET6) {
-			struct sockaddr_in6 *in = (struct sockaddr_in6 *) addr;
-			in->sin6_addr = in6addr_any;
-			in->sin6_port = htons(listen_port);
-			*addr_len = sizeof(struct sockaddr_in6);
-                        goto out;
-		} else if (addr->sa_family == AF_INET) {
-			struct sockaddr_in *in = (struct sockaddr_in *) addr;
-			in->sin_addr.s_addr = htonl(INADDR_ANY);
-			in->sin_port = htons(listen_port);
-			*addr_len = sizeof(struct sockaddr_in);
-			goto out;
-		}
-	}
-
-        memset (service, 0, sizeof (service));
-        sprintf (service, "%d", listen_port);
-
-        memset (&hints, 0, sizeof (hints));
-        hints.ai_family = addr->sa_family;
-        hints.ai_socktype = SOCK_STREAM;
-        hints.ai_flags    = AI_ADDRCONFIG | AI_PASSIVE;
-
-        ret = getaddrinfo(listen_host, service, &hints, &res);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "getaddrinfo failed for host %s, service %s (%s)", 
-                        listen_host, service, gai_strerror (ret));
-                ret = -1;
-                goto out;
-        }
-
-        memcpy (addr, res->ai_addr, res->ai_addrlen);
-        *addr_len = res->ai_addrlen;
-
-        freeaddrinfo (res);
-
-out:
-        return ret;
-}
-
-int32_t
-gf_client_bind (transport_t *this,
-             struct sockaddr *sockaddr,
-             socklen_t *sockaddr_len,
-             int sock)
-{
-        int ret = 0;
-
-        *sockaddr_len = sizeof (struct sockaddr_in6);
-        switch (sockaddr->sa_family)
-        {
-        case AF_INET_SDP:
-        case AF_INET:
-                *sockaddr_len = sizeof (struct sockaddr_in);
-
-        case AF_INET6:
-                ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr, 
-                                                       *sockaddr_len, CLIENT_PORT_CEILING);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "cannot bind inet socket (%d) to port less than %d (%s)", 
-                                sock, CLIENT_PORT_CEILING, strerror (errno));
-                        ret = 0;
-                }
-                break;
-
-        case AF_UNIX:
-                *sockaddr_len = sizeof (struct sockaddr_un);
-                ret = af_unix_client_bind (this, (struct sockaddr *)sockaddr, 
-                                           *sockaddr_len, sock);
-                break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address family %d", sockaddr->sa_family);
-                ret = -1;
-                break;
-        }
-
-        return ret;
-}
-
-int32_t
-gf_socket_client_get_remote_sockaddr (transport_t *this,
-                                   struct sockaddr *sockaddr,
-                                   socklen_t *sockaddr_len,
-                                   sa_family_t *sa_family)
-{
-        int32_t ret = 0;
-
-        if ((sockaddr == NULL) || (sockaddr_len == NULL)
-            || (sa_family == NULL)) {
-                ret = -1;
-                goto err;
-        }
-
-
-        ret = client_fill_address_family (this, &sockaddr->sa_family);
-        if (ret) {
-                ret = -1;
-                goto err;
-        }
- 
-        *sa_family = sockaddr->sa_family;
-
-        switch (sockaddr->sa_family)
-        {
-        case AF_INET_SDP:
-                sockaddr->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        case AF_UNSPEC:
-                ret = af_inet_client_get_remote_sockaddr (this, sockaddr,
-                                                          sockaddr_len);
-                break;
-
-        case AF_UNIX:
-                ret = af_unix_client_get_remote_sockaddr (this, sockaddr,
-                                                          sockaddr_len);
-                break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address-family %d", sockaddr->sa_family);
-                ret = -1;
-        }
-  
-        if (*sa_family == AF_UNSPEC) {
-                *sa_family = sockaddr->sa_family;
-        }
-
-err:
-        return ret;
-}
-
-
-static int32_t
-server_fill_address_family (transport_t *this, sa_family_t *sa_family)
-{
-        data_t  *address_family_data = NULL;
-        int32_t  ret                 = -1;
-        
-        if (sa_family == NULL) {
-                goto out;
-        }
-
-        address_family_data = dict_get (this->xl->options, 
-                                        "transport.address-family");
-        if (address_family_data) {
-                char *address_family = NULL;
-                address_family = data_to_str (address_family_data);
-
-                if (!strcasecmp (address_family, "inet")) {
-                        *sa_family = AF_INET;
-                } else if (!strcasecmp (address_family, "inet6")) {
-                        *sa_family = AF_INET6;
-                } else if (!strcasecmp (address_family, "inet-sdp")) {
-                        *sa_family = AF_INET_SDP;
-                } else if (!strcasecmp (address_family, "unix")) {
-                        *sa_family = AF_UNIX;
-                } else if (!strcasecmp (address_family, "inet/inet6")
-                           || !strcasecmp (address_family, "inet6/inet")) {
-                        *sa_family = AF_UNSPEC;
-                } else {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "unknown address family (%s) specified", address_family);
-                        goto out;
-                }
-        } else {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "option address-family not specified, defaulting to inet/inet6");
-                *sa_family = AF_UNSPEC;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-int32_t
-gf_socket_server_get_local_sockaddr (transport_t *this, struct sockaddr *addr,
-                                  socklen_t *addr_len, sa_family_t *sa_family)
-{
-        int32_t ret = -1;
-
-        if ((addr == NULL) || (addr_len == NULL) || (sa_family == NULL)) {
-                goto err;
-        }
-
-        ret = server_fill_address_family (this, &addr->sa_family);
-        if (ret == -1) {
-                goto err;
-        }
-
-        *sa_family = addr->sa_family;
-
-        switch (addr->sa_family)
-        {
-        case AF_INET_SDP:
-                addr->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        case AF_UNSPEC:
-                ret = af_inet_server_get_local_sockaddr (this, addr, addr_len);
-                break;
-
-        case AF_UNIX:
-                ret = af_unix_server_get_local_sockaddr (this, addr, addr_len);
-                break;
-        }
-
-        if (*sa_family == AF_UNSPEC) {
-                *sa_family = addr->sa_family;
-        }
-
-err:
-        return ret;
-}
-
-static int32_t
-fill_inet6_inet_identifiers (transport_t *this, struct sockaddr_storage *addr,
-                             int32_t addr_len, char *identifier)
-{
-        int32_t ret = 0, tmpaddr_len = 0;
-        char service[NI_MAXSERV], host[NI_MAXHOST];
-        struct sockaddr_storage tmpaddr;
-
-        memset (&tmpaddr, 0, sizeof (tmpaddr));
-        tmpaddr = *addr;
-        tmpaddr_len = addr_len;
-
-        if (((struct sockaddr *) &tmpaddr)->sa_family == AF_INET6) {
-                int32_t one_to_four, four_to_eight, twelve_to_sixteen;
-                int16_t eight_to_ten, ten_to_twelve;
-    
-                one_to_four = four_to_eight = twelve_to_sixteen = 0;
-                eight_to_ten = ten_to_twelve = 0;
-    
-                one_to_four = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr32[0];
-                four_to_eight = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr32[1];
-#ifdef GF_SOLARIS_HOST_OS
-                eight_to_ten = S6_ADDR16(((struct sockaddr_in6 *) &tmpaddr)->sin6_addr)[4];
-#else
-                eight_to_ten = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr16[4];
-#endif
-
-#ifdef GF_SOLARIS_HOST_OS
-                ten_to_twelve = S6_ADDR16(((struct sockaddr_in6 *) &tmpaddr)->sin6_addr)[5];
-#else
-                ten_to_twelve = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr16[5];
-#endif
-
-                twelve_to_sixteen = ((struct sockaddr_in6 *) &tmpaddr)->sin6_addr.s6_addr32[3];
-
-                /* ipv4 mapped ipv6 address has
-                   bits 0-80: 0
-                   bits 80-96: 0xffff
-                   bits 96-128: ipv4 address 
-                */
- 
-                if (one_to_four == 0 &&
-                    four_to_eight == 0 &&
-                    eight_to_ten == 0 &&
-                    ten_to_twelve == -1) {
-                        struct sockaddr_in *in_ptr = (struct sockaddr_in *)&tmpaddr;
-                        memset (&tmpaddr, 0, sizeof (tmpaddr));
-      
-                        in_ptr->sin_family = AF_INET;
-                        in_ptr->sin_port = ((struct sockaddr_in6 *)addr)->sin6_port;
-                        in_ptr->sin_addr.s_addr = twelve_to_sixteen;
-                        tmpaddr_len = sizeof (*in_ptr);
-                }
-        }
-
-        ret = getnameinfo ((struct sockaddr *) &tmpaddr,
-                           tmpaddr_len,
-                           host, sizeof (host),
-                           service, sizeof (service),
-                           NI_NUMERICHOST | NI_NUMERICSERV);
-        if (ret != 0) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "getnameinfo failed (%s)", gai_strerror (ret));
-        }
-
-        sprintf (identifier, "%s:%s", host, service);
-
-        return ret;
-}
-
-int32_t
-gf_get_transport_identifiers (transport_t *this)
-{
-        int32_t ret = 0;
-        char is_inet_sdp = 0;
-
-        switch (((struct sockaddr *) &this->myinfo.sockaddr)->sa_family)
-        {
-        case AF_INET_SDP:
-                is_inet_sdp = 1;
-                ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET;
-
-        case AF_INET:
-        case AF_INET6:
-        {
-                ret = fill_inet6_inet_identifiers (this, 
-                                                   &this->myinfo.sockaddr, 
-                                                   this->myinfo.sockaddr_len,
-                                                   this->myinfo.identifier);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "cannot fill inet/inet6 identifier for server");
-                        goto err;
-                }
-
-                ret = fill_inet6_inet_identifiers (this,
-                                                   &this->peerinfo.sockaddr,
-                                                   this->peerinfo.sockaddr_len,
-                                                   this->peerinfo.identifier);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "cannot fill inet/inet6 identifier for client");
-                        goto err;
-                }
-
-                if (is_inet_sdp) {
-                        ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET_SDP;
-                }
-        }
-        break;
-
-        case AF_UNIX:
-        {
-                struct sockaddr_un *sunaddr = NULL;
-
-                sunaddr = (struct sockaddr_un *) &this->myinfo.sockaddr;
-                strcpy (this->myinfo.identifier, sunaddr->sun_path);
-
-                sunaddr = (struct sockaddr_un *) &this->peerinfo.sockaddr;
-                strcpy (this->peerinfo.identifier, sunaddr->sun_path);
-        }
-        break;
-
-        default:
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "unknown address family (%d)", 
-                        ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family);
-                ret = -1;
-                break;
-        }
-
-err:
-        return ret;
-}
diff --git a/xlators/protocol/legacy/transport/socket/src/name.h b/xlators/protocol/legacy/transport/socket/src/name.h
deleted file mode 100644
index 789cf13ef..000000000
--- a/xlators/protocol/legacy/transport/socket/src/name.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SOCKET_NAME_H
-#define _SOCKET_NAME_H
-
-#include "compat.h"
-
-int32_t
-gf_client_bind (transport_t *this,
-                struct sockaddr *sockaddr,
-                socklen_t *sockaddr_len,
-                int sock);
-
-int32_t
-gf_socket_client_get_remote_sockaddr (transport_t *this,
-                                      struct sockaddr *sockaddr,
-                                      socklen_t *sockaddr_len,
-                                      sa_family_t *sa_family);
-
-int32_t
-gf_socket_server_get_local_sockaddr (transport_t *this, struct sockaddr *addr,
-                                     socklen_t *addr_len, sa_family_t *sa_family);
-
-int32_t
-gf_get_transport_identifiers (transport_t *this);
-
-#endif /* _SOCKET_NAME_H */
diff --git a/xlators/protocol/legacy/transport/socket/src/socket-mem-types.h b/xlators/protocol/legacy/transport/socket/src/socket-mem-types.h
deleted file mode 100644
index 937b9d7a8..000000000
--- a/xlators/protocol/legacy/transport/socket/src/socket-mem-types.h
+++ /dev/null
@@ -1,36 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __SOCKET_MEM_TYPES_H__
-#define __SOCKET_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_socket_mem_types_ {
-        gf_socket_mt_socket_private_t = gf_common_mt_end + 1,
-        gf_socket_mt_ioq,
-        gf_socket_mt_transport_t,
-        gf_socket_mt_socket_local_t,
-        gf_socket_mt_char,
-        gf_socket_mt_end
-};
-#endif
-
diff --git a/xlators/protocol/legacy/transport/socket/src/socket.c b/xlators/protocol/legacy/transport/socket/src/socket.c
deleted file mode 100644
index 9272b0584..000000000
--- a/xlators/protocol/legacy/transport/socket/src/socket.c
+++ /dev/null
@@ -1,1625 +0,0 @@
-/*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "socket.h"
-#include "name.h"
-#include "dict.h"
-#include "transport.h"
-#include "logging.h"
-#include "xlator.h"
-#include "byte-order.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-
-#include <fcntl.h>
-#include <errno.h>
-#include <netinet/tcp.h>
-
-
-#define GF_LOG_ERRNO(errno) ((errno == ENOTCONN) ? GF_LOG_DEBUG : GF_LOG_ERROR)
-#define SA(ptr) ((struct sockaddr *)ptr)
-
-static int socket_init (transport_t *this);
-
-/*
- * return value:
- *   0 = success (completed)
- *  -1 = error
- * > 0 = incomplete
- */
-
-static int
-__socket_rwv (transport_t *this, struct iovec *vector, int count,
-              struct iovec **pending_vector, int *pending_count,
-              int write)
-{
-        socket_private_t *priv = NULL;
-        int               sock = -1;
-        int               ret = -1;
-        struct iovec     *opvector = NULL;
-        int               opcount = 0;
-        int               moved = 0;
-
-        priv = this->private;
-        sock = priv->sock;
-
-	opvector = vector;
-	opcount  = count;
-
-        while (opcount) {
-                if (write) {
-                        ret = writev (sock, opvector, opcount);
-
-                        if (ret == 0 || (ret == -1 && errno == EAGAIN)) {
-                                /* done for now */
-                                break;
-                        }
-                } else {
-                        ret = readv (sock, opvector, opcount);
-
-                        if (ret == -1 && errno == EAGAIN) {
-                                /* done for now */
-                                break;
-                        }
-                }
-
-                if (ret == 0) {
-                        /* Mostly due to 'umount' in client */
-                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                "EOF from peer %s", this->peerinfo.identifier);
-                        opcount = -1;
-                        errno = ENOTCONN;
-                        break;
-                }
-
-                if (ret == -1) {
-                        if (errno == EINTR)
-                                continue;
-
-                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                "%s failed (%s)", write ? "writev" : "readv",
-                                strerror (errno));
-                        opcount = -1;
-                        break;
-                }
-
-                moved = 0;
-
-                while (moved < ret) {
-                        if ((ret - moved) >= opvector[0].iov_len) {
-                                moved += opvector[0].iov_len;
-                                opvector++;
-                                opcount--;
-                        } else {
-                                opvector[0].iov_len -= (ret - moved);
-                                opvector[0].iov_base += (ret - moved);
-                                moved += (ret - moved);
-                        }
-                        while (opcount && !opvector[0].iov_len) {
-                                opvector++;
-                                opcount--;
-                        }
-                }
-        }
-
-        if (pending_vector)
-                *pending_vector = opvector;
-
-        if (pending_count)
-                *pending_count = opcount;
-
-        return opcount;
-}
-
-
-static int
-__socket_readv (transport_t *this, struct iovec *vector, int count,
-                struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-
-        ret = __socket_rwv (this, vector, count,
-			    pending_vector, pending_count, 0);
-
-        return ret;
-}
-
-
-static int
-__socket_writev (transport_t *this, struct iovec *vector, int count,
-                 struct iovec **pending_vector, int *pending_count)
-{
-        int ret = -1;
-
-        ret = __socket_rwv (this, vector, count,
-			    pending_vector, pending_count, 1);
-
-        return ret;
-}
-
-
-static int
-__socket_disconnect (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        if (priv->sock != -1) {
-                ret = shutdown (priv->sock, SHUT_RDWR);
-                priv->connected = -1;
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "shutdown() returned %d. set connection state to -1",
-                        ret);
-        }
-
-        return ret;
-}
-
-
-static int
-__socket_server_bind (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-	int               opt = 1;
-
-	priv = this->private;
-
-        ret = setsockopt (priv->sock, SOL_SOCKET, SO_REUSEADDR,
-			  &opt, sizeof (opt));
-
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "setsockopt() for SO_REUSEADDR failed (%s)",
-                        strerror (errno));
-        }
-
-        ret = bind (priv->sock, (struct sockaddr *)&this->myinfo.sockaddr,
-		    this->myinfo.sockaddr_len);
-
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "binding to %s failed: %s",
-                        this->myinfo.identifier, strerror (errno));
-                if (errno == EADDRINUSE) {
-                        gf_log (this->xl->name, GF_LOG_ERROR, 
-                                "Port is already in use");
-                }
-        }
-
-        return ret;
-}
-
-
-static int
-__socket_nonblock (int fd)
-{
-        int flags = 0;
-        int ret = -1;
-
-        flags = fcntl (fd, F_GETFL);
-
-        if (flags != -1)
-                ret = fcntl (fd, F_SETFL, flags | O_NONBLOCK);
-
-        return ret;
-}
-
-
-static int
-__socket_nodelay (int fd)
-{
-        int     on = 1;
-        int     ret = -1;
-
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_NODELAY,
-			  &on, sizeof (on));
-        if (!ret)
-                gf_log ("", GF_LOG_TRACE,
-                        "NODELAY enabled for socket %d", fd);
-
-        return ret;
-}
-
-
-static int
-__socket_keepalive (int fd, int keepalive_intvl)
-{
-        int     on = 1;
-        int     ret = -1;
-
-        ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof (on));
-        if (ret == -1)
-                goto err;
-
-        if (keepalive_intvl == GF_USE_DEFAULT_KEEPALIVE)
-                goto done;
-
-#ifndef GF_LINUX_HOST_OS
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPALIVE, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
-        if (ret == -1)
-                goto err;
-#else
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
-        if (ret == -1)
-                goto err;
-
-        ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPINTVL, &keepalive_intvl,
-                          sizeof (keepalive_intvl));
-        if (ret == -1)
-                goto err;
-#endif
-
-done:
-        gf_log ("", GF_LOG_TRACE, "Keep-alive enabled for socket %d, interval "
-                "%d", fd, keepalive_intvl);
-
-err:
-        return ret;
-}
-
-
-static int
-__socket_connect_finish (int fd)
-{
-        int       ret = -1;
-        int       optval = 0;
-        socklen_t optlen = sizeof (int);
-
-        ret = getsockopt (fd, SOL_SOCKET, SO_ERROR, (void *)&optval, &optlen);
-
-        if (ret == 0 && optval) {
-                errno = optval;
-                ret = -1;
-        }
-
-        return ret;
-}
-
-
-static void
-__socket_reset (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-
-        priv = this->private;
-
-        /* TODO: use mem-pool on incoming data */
-
-        if (priv->incoming.hdr_p)
-                GF_FREE (priv->incoming.hdr_p);
-
-        if (priv->incoming.iobuf)
-                iobuf_unref (priv->incoming.iobuf);
-
-        memset (&priv->incoming, 0, sizeof (priv->incoming));
-
-        event_unregister (this->xl->ctx->event_pool, priv->sock, priv->idx);
-        close (priv->sock);
-        priv->sock = -1;
-        priv->idx = -1;
-        priv->connected = -1;
-}
-
-
-static struct ioq *
-__socket_ioq_new (transport_t *this, char *buf, int len,
-                  struct iovec *vector, int count, struct iobref *iobref)
-{
-        socket_private_t *priv = NULL;
-        struct ioq       *entry = NULL;
-
-        priv = this->private;
-
-        /* TODO: use mem-pool */
-        entry = GF_CALLOC (1, sizeof (*entry),
-                           gf_common_mt_ioq);
-        if (!entry)
-                return NULL;
-
-        GF_ASSERT (count <= (MAX_IOVEC-2));
-
-        entry->header.colonO[0] = ':';
-        entry->header.colonO[1] = 'O';
-        entry->header.colonO[2] = '\0';
-        entry->header.version   = 42;
-        entry->header.size1     = hton32 (len);
-        entry->header.size2     = hton32 (iov_length (vector, count));
-
-        entry->vector[0].iov_base = &entry->header;
-        entry->vector[0].iov_len  = sizeof (entry->header);
-        entry->count++;
-
-        entry->vector[1].iov_base = buf;
-        entry->vector[1].iov_len  = len;
-        entry->count++;
-
-        if (vector && count) {
-                memcpy (&entry->vector[2], vector, sizeof (*vector) * count);
-                entry->count += count;
-        }
-
-        entry->pending_vector = entry->vector;
-        entry->pending_count  = entry->count;
-
-        if (iobref)
-                entry->iobref = iobref_ref (iobref);
-
-        entry->buf = buf;
-
-        INIT_LIST_HEAD (&entry->list);
-
-        return entry;
-}
-
-
-static void
-__socket_ioq_entry_free (struct ioq *entry)
-{
-        list_del_init (&entry->list);
-        if (entry->iobref)
-                iobref_unref (entry->iobref);
-
-        /* TODO: use mem-pool */
-        GF_FREE (entry->buf);
-
-        /* TODO: use mem-pool */
-        GF_FREE (entry);
-}
-
-
-static void
-__socket_ioq_flush (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        struct ioq       *entry = NULL;
-
-        priv = this->private;
-
-        while (!list_empty (&priv->ioq)) {
-                entry = priv->ioq_next;
-                __socket_ioq_entry_free (entry);
-        }
-
-        return;
-}
-
-
-static int
-__socket_ioq_churn_entry (transport_t *this, struct ioq *entry)
-{
-        int ret = -1;
-
-        ret = __socket_writev (this, entry->pending_vector,
-			       entry->pending_count,
-                               &entry->pending_vector,
-			       &entry->pending_count);
-
-        if (ret == 0) {
-                /* current entry was completely written */
-                GF_ASSERT (entry->pending_count == 0);
-                __socket_ioq_entry_free (entry);
-        }
-
-        return ret;
-}
-
-
-static int
-__socket_ioq_churn (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = 0;
-        struct ioq       *entry = NULL;
-
-        priv = this->private;
-
-        while (!list_empty (&priv->ioq)) {
-                /* pick next entry */
-                entry = priv->ioq_next;
-
-                ret = __socket_ioq_churn_entry (this, entry);
-
-                if (ret != 0)
-                        break;
-        }
-
-        if (list_empty (&priv->ioq)) {
-                /* all pending writes done, not interested in POLLOUT */
-                priv->idx = event_select_on (this->xl->ctx->event_pool,
-					     priv->sock, priv->idx, -1, 0);
-        }
-
-        return ret;
-}
-
-
-static int
-socket_event_poll_err (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                __socket_ioq_flush (this);
-                __socket_reset (this);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        xlator_notify (this->xl, GF_EVENT_POLLERR, this);
-
-        return ret;
-}
-
-
-static int
-socket_event_poll_out (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->connected == 1) {
-                        ret = __socket_ioq_churn (this);
-
-                        if (ret == -1) {
-                                __socket_disconnect (this);
-                        }
-                }
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        xlator_notify (this->xl, GF_EVENT_POLLOUT, this);
-
-        return ret;
-}
-
-
-static int
-__socket_proto_validate_header (transport_t *this,
-				struct socket_header *header,
-				size_t *size1_p, size_t *size2_p)
-{
-        size_t size1 = 0;
-	size_t size2 = 0;
-
-        if (strcmp (header->colonO, ":O")) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "socket header signature does not match :O (%x.%x.%x)",
-                        header->colonO[0], header->colonO[1],
-			header->colonO[2]);
-                return -1;
-        }
-
-        if (header->version != 42) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "socket header version does not match 42 != %d",
-			header->version);
-                return -1;
-        }
-
-        size1 = ntoh32 (header->size1);
-        size2 = ntoh32 (header->size2);
-
-        if (size1 <= 0 || size1 > 1048576) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "socket header has incorrect size1=%"GF_PRI_SIZET,
-			size1);
-                return -1;
-        }
-
-        if (size2 > (131072)) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "socket header has incorrect size2=%"GF_PRI_SIZET,
-			size2);
-                return -1;
-        }
-
-        if (size1_p)
-                *size1_p = size1;
-
-        if (size2_p)
-                *size2_p = size2;
-
-        return 0;
-}
-
-
-
-/* socket protocol state machine */
-
-static int
-__socket_proto_state_machine (transport_t *this)
-{
-        int                   ret = -1;
-        socket_private_t     *priv = NULL;
-        size_t                size1 = 0;
-	size_t                size2 = 0;
-        int                   previous_state = -1;
-	struct socket_header *hdr = NULL;
-        struct iobuf         *iobuf = NULL;
-
-
-        priv = this->private;
-
-	while (priv->incoming.state != SOCKET_PROTO_STATE_COMPLETE) {
-		/* debug check against infinite loops */
-		if (previous_state == priv->incoming.state) {
-			gf_log (this->xl->name, GF_LOG_DEBUG,
-				"state did not change! (%d) breaking",
-				previous_state);
-			ret = -1;
-			goto unlock;
-		}
-		previous_state = priv->incoming.state;
-
-		switch (priv->incoming.state) {
-
-		case SOCKET_PROTO_STATE_NADA:
-			priv->incoming.pending_vector =
-				priv->incoming.vector;
-
-			priv->incoming.pending_vector->iov_base =
-				&priv->incoming.header;
-
-			priv->incoming.pending_vector->iov_len  =
-				sizeof (struct socket_header);
-
-			priv->incoming.state =
-				SOCKET_PROTO_STATE_HEADER_COMING;
-			break;
-
-		case SOCKET_PROTO_STATE_HEADER_COMING:
-
-			ret = __socket_readv (this,
-					      priv->incoming.pending_vector, 1,
-					      &priv->incoming.pending_vector,
-					      NULL);
-			if (ret == 0) {
-				priv->incoming.state =
-					SOCKET_PROTO_STATE_HEADER_CAME;
-				break;
-			}
-
-			if (ret == -1) {
-				gf_log (this->xl->name, GF_LOG_TRACE,
-					"read (%s) in state %d (%s)",
-					strerror (errno),
-					SOCKET_PROTO_STATE_HEADER_COMING, 
-					this->peerinfo.identifier);
-				goto unlock;
-			}
-
-			if (ret > 0) {
-				gf_log (this->xl->name, GF_LOG_TRACE,
-					"partial header read on NB socket.");
-				goto unlock;
-			}
-			break;
-
-		case SOCKET_PROTO_STATE_HEADER_CAME:
-			hdr = &priv->incoming.header;
-			ret = __socket_proto_validate_header (this, hdr,
-							      &size1, &size2);
-
-			if (ret == -1) {
-				gf_log (this->xl->name, GF_LOG_ERROR,
-					"socket header validate failed (%s). "
-					"possible mismatch of transport-type "
-					"between server and client volumes, "
-					"or version mismatch",
-					this->peerinfo.identifier);
-                                        goto unlock;
-                        }
-
-                        priv->incoming.hdrlen = size1;
-                        priv->incoming.buflen = size2;
-
-                        /* TODO: use mem-pool */
-                        priv->incoming.hdr_p  = GF_MALLOC (size1, 
-                                                           gf_common_mt_char);
-                        if (size2) {
-                                /* TODO: sanity check size2 < page size
-                                 */
-                                iobuf = iobuf_get (this->xl->ctx->iobuf_pool);
-                                if (!iobuf) {
-                                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                                "unable to allocate IO buffer "
-                                                "for peer %s",
-                                                this->peerinfo.identifier);
-                                        ret = -ENOMEM;
-                                        goto unlock;
-                                }
-                                priv->incoming.iobuf = iobuf;
-                                priv->incoming.buf_p = iobuf->ptr;
-                        }
-
-                        priv->incoming.vector[0].iov_base =
-                                priv->incoming.hdr_p;
-
-                        priv->incoming.vector[0].iov_len  = size1;
-
-                        priv->incoming.vector[1].iov_base =
-                                priv->incoming.buf_p;
-
-                        priv->incoming.vector[1].iov_len  = size2;
-                        priv->incoming.count = size2 ? 2 : 1;
-
-                        priv->incoming.pending_vector =
-                                priv->incoming.vector;
-
-                        priv->incoming.pending_count  =
-                                priv->incoming.count;
-
-                        priv->incoming.state =
-                                SOCKET_PROTO_STATE_DATA_COMING;
-                        break;
-
-		case SOCKET_PROTO_STATE_DATA_COMING:
-
-			ret = __socket_readv (this,
-					      priv->incoming.pending_vector,
-					      priv->incoming.pending_count,
-					      &priv->incoming.pending_vector,
-					      &priv->incoming.pending_count);
-			if (ret == 0) {
-				priv->incoming.state =
-					SOCKET_PROTO_STATE_DATA_CAME;
-				break;
-			}
-
-			if (ret == -1) {
-				gf_log (this->xl->name, GF_LOG_DEBUG,
-					"read (%s) in state %d (%s)",
-					strerror (errno),
-					SOCKET_PROTO_STATE_DATA_COMING,
-					this->peerinfo.identifier);
-				goto unlock;
-			}
-
-			if (ret > 0) {
-				gf_log (this->xl->name, GF_LOG_TRACE,
-					"partial data read on NB socket");
-                                        goto unlock;
-			}
-			break;
-
-		case SOCKET_PROTO_STATE_DATA_CAME:
-			memset (&priv->incoming.vector, 0,
-				sizeof (priv->incoming.vector));
-			priv->incoming.pending_vector = NULL;
-			priv->incoming.pending_count  = 0;
-			priv->incoming.state = SOCKET_PROTO_STATE_COMPLETE;
-			break;
-
-		case SOCKET_PROTO_STATE_COMPLETE:
-			/* not reached */
-			break;
-
-		default:
-			gf_log (this->xl->name, GF_LOG_DEBUG,
-				"undefined state reached: %d",
-				priv->incoming.state);
-                                goto unlock;
-		}
-	}
-unlock:
-
-        return ret;
-}
-
-
-static int
-socket_proto_state_machine (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-	int               ret = 0;
-
-	priv = this->private;
-
-	pthread_mutex_lock (&priv->lock);
-	{
-		ret = __socket_proto_state_machine (this);
-	}
-        pthread_mutex_unlock (&priv->lock);
-
-	return ret;
-}
-
-
-static int
-socket_event_poll_in (transport_t *this)
-{
-        int ret = -1;
-
-        ret = socket_proto_state_machine (this);
-
-        /* call POLLIN on xlator even if complete block is not received,
-           just to keep the last_received timestamp ticking */
-
-        if (ret == 0)
-                ret = xlator_notify (this->xl, GF_EVENT_POLLIN, this);
-
-        return ret;
-}
-
-
-static int
-socket_connect_finish (transport_t *this)
-{
-        int               ret = -1;
-        socket_private_t *priv = NULL;
-        int               event = -1;
-        char              notify_xlator = 0;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-		if (priv->connected)
-			goto unlock;
-
-		ret = __socket_connect_finish (priv->sock);
-
-		if (ret == -1 && errno == EINPROGRESS)
-			ret = 1;
-
-		if (ret == -1 && errno != EINPROGRESS) {
-			if (!priv->connect_finish_log) {
-				gf_log (this->xl->name, GF_LOG_ERROR,
-					"connection to %s failed (%s)",
-                                        this->peerinfo.identifier,
-					strerror (errno));
-				priv->connect_finish_log = 1;
-			}
-			__socket_disconnect (this);
-			notify_xlator = 1;
-			event = GF_EVENT_POLLERR;
-			goto unlock;
-		}
-
-		if (ret == 0) {
-			notify_xlator = 1;
-
-			this->myinfo.sockaddr_len =
-				sizeof (this->myinfo.sockaddr);
-
-			ret = getsockname (priv->sock,
-					   SA (&this->myinfo.sockaddr),
-					   &this->myinfo.sockaddr_len);
-			if (ret == -1) {
-				gf_log (this->xl->name, GF_LOG_DEBUG,
-					"getsockname on (%d) failed (%s)", 
-					priv->sock, strerror (errno));
-				__socket_disconnect (this);
-				event = GF_EVENT_POLLERR;
-				goto unlock;
-			}
-
-			priv->connected = 1;
-			priv->connect_finish_log = 0;
-			event = GF_EVENT_CHILD_UP;
-			gf_get_transport_identifiers (this);
-		}
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        if (notify_xlator)
-                xlator_notify (this->xl, event, this);
-
-        return 0;
-}
-
-
-static int
-socket_event_handler (int fd, int idx, void *data,
-                      int poll_in, int poll_out, int poll_err)
-{
-        transport_t      *this = NULL;
-        socket_private_t *priv = NULL;
-        int               ret = 0;
-
-        this = data;
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                priv->idx = idx;
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        if (!priv->connected) {
-                ret = socket_connect_finish (this);
-        }
-
-        if (!ret && poll_out) {
-                ret = socket_event_poll_out (this);
-        }
-
-        if (!ret && poll_in) {
-                ret = socket_event_poll_in (this);
-        }
-
-        if (ret < 0 || poll_err) {
-                socket_event_poll_err (this);
-                transport_unref (this);
-        }
-
-        return 0;
-}
-
-
-static int
-socket_server_event_handler (int fd, int idx, void *data,
-                             int poll_in, int poll_out, int poll_err)
-{
-        transport_t             *this = NULL;
-        socket_private_t        *priv = NULL;
-        int                      ret = 0;
-        int                      new_sock = -1;
-        transport_t             *new_trans = NULL;
-        struct sockaddr_storage  new_sockaddr = {0, };
-        socklen_t                addrlen = sizeof (new_sockaddr);
-        socket_private_t        *new_priv = NULL;
-	glusterfs_ctx_t         *ctx = NULL;
-
-        this = data;
-        priv = this->private;
-	ctx  = this->xl->ctx;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                priv->idx = idx;
-
-                if (poll_in) {
-                        new_sock = accept (priv->sock, SA (&new_sockaddr),
-					   &addrlen);
-
-                        if (new_sock == -1)
-                                goto unlock;
-
-                        if (!priv->bio) {
-                                ret = __socket_nonblock (new_sock);
-
-                                if (ret == -1) {
-                                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                                "NBIO on %d failed (%s)",
-                                                new_sock, strerror (errno));
-                                        close (new_sock);
-                                        goto unlock;
-                                }
-                        }
-
-                        if (priv->nodelay) {
-                                ret = __socket_nodelay (new_sock);
-                                if (ret == -1) {
-                                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                                "setsockopt() failed for "
-                                                "NODELAY (%s)",
-                                                strerror (errno));
-                                }
-                        }
-
-                        if (priv->keepalive) {
-                                ret = __socket_keepalive (new_sock,
-                                                          priv->keepaliveintvl);
-                                if (ret == -1)
-                                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                                "Failed to set keep-alive: %s",
-                                                strerror (errno));
-                        }
-
-                        new_trans = GF_CALLOC (1, sizeof (*new_trans), 
-                                               gf_common_mt_transport_t);
-                        new_trans->xl = this->xl;
-                        new_trans->fini = this->fini;
-
-                        memcpy (&new_trans->peerinfo.sockaddr, &new_sockaddr,
-				addrlen);
-                        new_trans->peerinfo.sockaddr_len = addrlen;
-
-                        new_trans->myinfo.sockaddr_len =
-				sizeof (new_trans->myinfo.sockaddr);
-
-                        ret = getsockname (new_sock, 
-                                           SA (&new_trans->myinfo.sockaddr),
-                                           &new_trans->myinfo.sockaddr_len);
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_DEBUG,
-                                        "getsockname on %d failed (%s)", 
-                                        new_sock, strerror (errno));
-                                close (new_sock);
-                                goto unlock;
-                        }
-
-                        gf_get_transport_identifiers (new_trans);
-                        socket_init (new_trans);
-                        new_trans->ops = this->ops;
-                        new_trans->init = this->init;
-                        new_trans->fini = this->fini;
-
-                        new_priv = new_trans->private;
-
-                        pthread_mutex_lock (&new_priv->lock);
-                        {
-                                new_priv->sock = new_sock;
-                                new_priv->connected = 1;
-        
-                                transport_ref (new_trans);
-                                new_priv->idx =
-					event_register (ctx->event_pool,
-							new_sock,
-							socket_event_handler,
-							new_trans, 1, 0);
-
-                                if (new_priv->idx == -1)
-                                        ret = -1;
-                        }
-                        pthread_mutex_unlock (&new_priv->lock);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-static int
-socket_disconnect (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                ret = __socket_disconnect (this);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-static int
-socket_connect (transport_t *this)
-{
-        int                      ret = -1;
-	int                      sock = -1;
-        socket_private_t        *priv = NULL;
-        struct sockaddr_storage  sockaddr = {0, };
-        socklen_t                sockaddr_len = 0;
-	glusterfs_ctx_t         *ctx = NULL;
-        sa_family_t              sa_family = {0, };
-
-        priv = this->private;
-	ctx = this->xl->ctx;
-
-        if (!priv) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "connect() called on uninitialized transport");
-                goto err;
-        }
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                sock = priv->sock;
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        if (sock != -1) {
-                gf_log (this->xl->name, GF_LOG_TRACE,
-                        "connect () called on transport already connected");
-                ret = 0;
-                goto err;
-        }
-
-        ret = gf_socket_client_get_remote_sockaddr (this, SA (&sockaddr),
-                                                 &sockaddr_len, &sa_family);
-        if (ret == -1) {
-                /* logged inside client_get_remote_sockaddr */
-                goto err;
-        }
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->sock != -1) {
-                        gf_log (this->xl->name, GF_LOG_TRACE,
-                                "connect() -- already connected");
-                        goto unlock;
-                }
-
-                memcpy (&this->peerinfo.sockaddr, &sockaddr, sockaddr_len);
-                this->peerinfo.sockaddr_len = sockaddr_len;
-
-                priv->sock = socket (sa_family, SOCK_STREAM, 0);
-                if (priv->sock == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "socket creation failed (%s)",
-				strerror (errno));
-                        goto unlock;
-                }
-
-                /* Cant help if setting socket options fails. We can continue
-                 * working nonetheless.
-                 */
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "setting receive window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "setting send window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-
-                if (priv->nodelay && priv->lowlat) {
-                        ret = __socket_nodelay (priv->sock);
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "setsockopt() failed for NODELAY (%s)",
-                                        strerror (errno));
-                        }
-                }
-
-                if (!priv->bio) {
-                        ret = __socket_nonblock (priv->sock);
-
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "NBIO on %d failed (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
-                        }
-                }
-
-                if (priv->keepalive) {
-                        ret = __socket_keepalive (priv->sock,
-                                                  priv->keepaliveintvl);
-                        if (ret == -1)
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "Failed to set keep-alive: %s",
-                                        strerror (errno));
-                }
-
-                SA (&this->myinfo.sockaddr)->sa_family =
-			SA (&this->peerinfo.sockaddr)->sa_family;
-
-                ret = gf_client_bind (this, SA (&this->myinfo.sockaddr),
-				   &this->myinfo.sockaddr_len, priv->sock);
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "client bind failed: %s", strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                ret = connect (priv->sock, SA (&this->peerinfo.sockaddr),
-			       this->peerinfo.sockaddr_len);
-
-                if (ret == -1 && errno != EINPROGRESS) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "connection attempt failed (%s)",
-				strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                priv->connected = 0;
-
-                transport_ref (this);
-
-                priv->idx = event_register (ctx->event_pool, priv->sock,
-                                            socket_event_handler, this, 1, 1);
-                if (priv->idx == -1)
-                        ret = -1;
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-err:
-        return ret;
-}
-
-
-static int
-socket_listen (transport_t *this)
-{
-        socket_private_t *       priv = NULL;
-        int                      ret = -1;
-	int                      sock = -1;
-        struct sockaddr_storage  sockaddr;
-        socklen_t                sockaddr_len;
-        peer_info_t             *myinfo = NULL;
-	glusterfs_ctx_t         *ctx = NULL;
-        sa_family_t              sa_family = {0, };
-
-	priv   = this->private;
-	myinfo = &this->myinfo;
-	ctx    = this->xl->ctx;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                sock = priv->sock;
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        if (sock != -1)  {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "already listening");
-                return ret;
-        }
-
-        ret = gf_socket_server_get_local_sockaddr (this, SA (&sockaddr),
-                                                &sockaddr_len, &sa_family);
-        if (ret == -1) {
-                return ret;
-        }
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->sock != -1) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "already listening");
-                        goto unlock;
-                }
-
-                memcpy (&myinfo->sockaddr, &sockaddr, sockaddr_len);
-                myinfo->sockaddr_len = sockaddr_len;
-
-                priv->sock = socket (sa_family, SOCK_STREAM, 0);
-
-                if (priv->sock == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "socket creation failed (%s)",
-				strerror (errno));
-                        goto unlock;
-                }
-
-                /* Cant help if setting socket options fails. We can continue
-                 * working nonetheless.
-                 */
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "setting receive window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-                if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
-                                &priv->windowsize,
-                                sizeof (priv->windowsize)) < 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                               "setting send window size failed: %d: %d: "
-                                "%s", priv->sock, priv->windowsize,
-                                strerror (errno));
-                }
-
-                if (priv->nodelay) {
-                        ret = __socket_nodelay (priv->sock);
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "setsockopt() failed for NODELAY (%s)",
-                                        strerror (errno));
-                        }
-                }
-
-                if (!priv->bio) {
-                        ret = __socket_nonblock (priv->sock);
-
-                        if (ret == -1) {
-                                gf_log (this->xl->name, GF_LOG_ERROR,
-                                        "NBIO on %d failed (%s)",
-                                        priv->sock, strerror (errno));
-                                close (priv->sock);
-                                priv->sock = -1;
-                                goto unlock;
-                        }
-                }
-
-                ret = __socket_server_bind (this);
-
-                if (ret == -1) {
-                        /* logged inside __socket_server_bind() */
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                ret = listen (priv->sock, 10);
-
-                if (ret == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "could not set socket %d to listen mode (%s)",
-				priv->sock, strerror (errno));
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-
-                transport_ref (this);
-
-                priv->idx = event_register (ctx->event_pool, priv->sock,
-                                            socket_server_event_handler,
-					    this, 1, 0);
-
-                if (priv->idx == -1) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "could not register socket %d with events",
-				priv->sock);
-                        ret = -1;
-                        close (priv->sock);
-                        priv->sock = -1;
-                        goto unlock;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-static int
-socket_receive (transport_t *this, char **hdr_p, size_t *hdrlen_p,
-                struct iobuf **iobuf_p)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->connected != 1) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "socket not connected to receive");
-                        goto unlock;
-                }
-
-                if (!hdr_p || !hdrlen_p || !iobuf_p) {
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "bad parameters %p %p %p",
-                                hdr_p, hdrlen_p, iobuf_p);
-                        goto unlock;
-                }
-
-                if (priv->incoming.state == SOCKET_PROTO_STATE_COMPLETE) {
-                        *hdr_p    = priv->incoming.hdr_p;
-                        *hdrlen_p = priv->incoming.hdrlen;
-                        *iobuf_p  = priv->incoming.iobuf;
-
-                        memset (&priv->incoming, 0, sizeof (priv->incoming));
-                        priv->incoming.state = SOCKET_PROTO_STATE_NADA;
-
-                        ret = 0;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-/* TODO: implement per transfer limit */
-static int
-socket_submit (transport_t *this, char *buf, int len,
-               struct iovec *vector, int count,
-               struct iobref *iobref)
-{
-        socket_private_t *priv = NULL;
-        int               ret = -1;
-        char              need_poll_out = 0;
-        char              need_append = 1;
-        struct ioq       *entry = NULL;
-	glusterfs_ctx_t  *ctx = NULL;
-
-        priv = this->private;
-	ctx  = this->xl->ctx;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                if (priv->connected != 1) {
-                        if (!priv->submit_log && !priv->connect_finish_log) {
-                                gf_log (this->xl->name, GF_LOG_DEBUG,
-                                        "not connected (priv->connected = %d)",
-                                        priv->connected);
-                                priv->submit_log = 1;
-                        }
-                        goto unlock;
-                }
-
-                priv->submit_log = 0;
-                entry = __socket_ioq_new (this, buf, len, vector, count, iobref);
-                if (!entry)
-                        goto unlock;
-
-                if (list_empty (&priv->ioq)) {
-                        ret = __socket_ioq_churn_entry (this, entry);
-
-                        if (ret == 0)
-                                need_append = 0;
-
-                        if (ret > 0)
-                                need_poll_out = 1;
-                }
-
-                if (need_append) {
-                        list_add_tail (&entry->list, &priv->ioq);
-                        ret = 0;
-                }
-
-                if (need_poll_out) {
-                        /* first entry to wait. continue writing on POLLOUT */
-                        priv->idx = event_select_on (ctx->event_pool,
-						     priv->sock,
-                                                     priv->idx, -1, 1);
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->lock);
-
-        return ret;
-}
-
-
-struct transport_ops tops = {
-        .listen     = socket_listen,
-        .connect    = socket_connect,
-        .disconnect = socket_disconnect,
-        .submit     = socket_submit,
-        .receive    = socket_receive
-};
-
-
-static int
-socket_init (transport_t *this)
-{
-        socket_private_t *priv = NULL;
-        gf_boolean_t      tmp_bool = 0;
-        uint64_t          windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE;
-        char             *optstr = NULL;
-        uint32_t          keepalive = 0;
-
-        if (this->private) {
-                gf_log (this->xl->name, GF_LOG_DEBUG,
-                        "double init attempted");
-                return -1;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), 
-                         gf_common_mt_socket_private_t);
-        if (!priv) {
-                gf_log (this->xl->name, GF_LOG_ERROR,
-                        "calloc (1, %"GF_PRI_SIZET") returned NULL",
-			sizeof (*priv));
-                return -1;
-        }
-
-        pthread_mutex_init (&priv->lock, NULL);
-
-        priv->sock = -1;
-        priv->idx = -1;
-        priv->connected = -1;
-
-        INIT_LIST_HEAD (&priv->ioq);
-
-        if (dict_get (this->xl->options, "non-blocking-io")) {
-                optstr = data_to_str (dict_get (this->xl->options,
-                                                          "non-blocking-io"));
-      
-                if (gf_string2boolean (optstr, &tmp_bool) == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "'non-blocking-io' takes only boolean options,"
-				" not taking any action");
-                        tmp_bool = 1;
-                }
-                priv->bio = 0;
-                if (!tmp_bool) {
-                        priv->bio = 1;
-                        gf_log (this->xl->name, GF_LOG_WARNING,
-                                "disabling non-blocking IO");
-                }
-        }
-
-        optstr = NULL;
-        
-        // By default, we enable NODELAY
-        priv->nodelay = 1;
-        if (dict_get (this->xl->options, "transport.socket.nodelay")) {
-                optstr = data_to_str (dict_get (this->xl->options,
-                                                "transport.socket.nodelay"));
-
-                if (gf_string2boolean (optstr, &tmp_bool) == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "'transport.socket.nodelay' takes only "
-                                 "boolean options, not taking any action");
-                        tmp_bool = 1;
-                }
-                if (!tmp_bool) {
-                        priv->nodelay = 0;
-                        gf_log (this->xl->name, GF_LOG_DEBUG,
-                                "disabling nodelay");
-                }
-        }
-
-
-        optstr = NULL;
-        if (dict_get_str (this->xl->options, "transport.window-size",
-                          &optstr) == 0) {
-                if (gf_string2bytesize (optstr, &windowsize) != 0) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "invalid number format: %s", optstr);
-                        return -1;
-                }
-        }
-
-        optstr = NULL;
-
-        if (dict_get_str (this->xl->options, "transport.socket.lowlat",
-                          &optstr) == 0) {
-                priv->lowlat = 1;
-        }
-
-        /* Enable Keep-alive by default. */
-        priv->keepalive = 1;
-        priv->keepaliveintvl = GF_USE_DEFAULT_KEEPALIVE;
-        if (dict_get_str (this->xl->options, "transport.socket.keepalive",
-                          &optstr) == 0) {
-                if (gf_string2boolean (optstr, &tmp_bool) == -1) {
-                        gf_log (this->xl->name, GF_LOG_ERROR,
-                                "'transport.socket.keepalive' takes only "
-                                 "boolean options, not taking any action");
-                        tmp_bool = 1;
-                }
-
-                if (!tmp_bool)
-                        priv->keepalive = 0;
-
-        }
-
-        if (dict_get_uint32 (this->xl->options,
-                             "transport.socket.keepalive-interval",
-                             &keepalive) == 0) {
-                priv->keepaliveintvl = keepalive;
-        }
-
-        priv->windowsize = (int)windowsize;
-        this->private = priv;
-
-        return 0;
-}
-
-
-void
-fini (transport_t *this)
-{
-        socket_private_t *priv = this->private;
-
-        if (!priv)
-                return;
-        this->private = NULL;
-        gf_log (this->xl->name, GF_LOG_TRACE,
-                "transport %p destroyed", this);
-
-        pthread_mutex_destroy (&priv->lock);
-        GF_FREE (priv);
-}
-
-
-int32_t
-init (transport_t *this)
-{
-        int ret = -1;
-
-        ret = socket_init (this);
-
-        if (ret == -1) {
-                gf_log (this->xl->name, GF_LOG_DEBUG, "socket_init() failed");
-        }
-  
-        return ret;
-}
-
-struct volume_options options[] = {
-        { .key   = {"remote-port", 
-                    "transport.remote-port",
-                    "transport.socket.remote-port"}, 
-          .type  = GF_OPTION_TYPE_INT 
-        },
-        { .key   = {"transport.socket.listen-port", "listen-port"}, 
-          .type  = GF_OPTION_TYPE_INT 
-        },
-        { .key   = {"transport.socket.bind-address", "bind-address" }, 
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS 
-        },
-        { .key   = {"transport.socket.connect-path", "connect-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.socket.bind-path", "bind-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = {"transport.socket.listen-path", "listen-path"}, 
-          .type  = GF_OPTION_TYPE_ANY 
-        },
-        { .key   = { "transport.address-family",
-                     "address-family" }, 
-          .value = {"inet", "inet6", "inet/inet6", "inet6/inet",
-                    "unix", "inet-sdp" },
-          .type  = GF_OPTION_TYPE_STR 
-        },
-
-        { .key   = {"non-blocking-io"}, 
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"transport.window-size"},
-          .type  = GF_OPTION_TYPE_SIZET,
-          .min   = GF_MIN_SOCKET_WINDOW_SIZE,
-          .max   = GF_MAX_SOCKET_WINDOW_SIZE,
-        },
-        { .key   = {"transport.socket.nodelay"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"transport.socket.lowlat"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"transport.socket.keepalive"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"transport.socket.keepalive-interval"},
-          .type  = GF_OPTION_TYPE_INT
-        },
-        { .key = {NULL} }
-};
-
diff --git a/xlators/protocol/legacy/transport/socket/src/socket.h b/xlators/protocol/legacy/transport/socket/src/socket.h
deleted file mode 100644
index e1dc9a91e..000000000
--- a/xlators/protocol/legacy/transport/socket/src/socket.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _SOCKET_H
-#define _SOCKET_H
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "event.h"
-#include "transport.h"
-#include "logging.h"
-#include "dict.h"
-#include "mem-pool.h"
-#include "socket-mem-types.h"
-
-#ifndef MAX_IOVEC
-#define MAX_IOVEC 16
-#endif /* MAX_IOVEC */
-
-#define GF_DEFAULT_SOCKET_LISTEN_PORT 6996
-
-/* This is the size set through setsockopt for
- * both the TCP receive window size and the
- * send buffer size.
- * Till the time iobuf size becomes configurable, this size is set to include
- * two iobufs + the GlusterFS protocol headers.
- * Linux allows us to over-ride the max values for the system.
- * Should we over-ride them? Because if we set a value larger than the default
- * setsockopt will fail. Having larger values might be beneficial for
- * IB links.
- */
-#define GF_DEFAULT_SOCKET_WINDOW_SIZE   (512 * GF_UNIT_KB)
-#define GF_MAX_SOCKET_WINDOW_SIZE       (1 * GF_UNIT_MB)
-#define GF_MIN_SOCKET_WINDOW_SIZE       (128 * GF_UNIT_KB)
-
-#define GF_USE_DEFAULT_KEEPALIVE        (-1)
-
-typedef enum {
-        SOCKET_PROTO_STATE_NADA = 0,
-        SOCKET_PROTO_STATE_HEADER_COMING,
-        SOCKET_PROTO_STATE_HEADER_CAME,
-        SOCKET_PROTO_STATE_DATA_COMING,
-        SOCKET_PROTO_STATE_DATA_CAME,
-        SOCKET_PROTO_STATE_COMPLETE,
-} socket_proto_state_t;
-
-struct socket_header {
-        char     colonO[3];
-        uint32_t size1;
-        uint32_t size2;
-        char     version;
-} __attribute__((packed));
-
-
-struct ioq {
-        union {
-                struct list_head list;
-                struct {
-                        struct ioq    *next;
-                        struct ioq    *prev;
-                };
-        };
-        struct socket_header  header;
-        struct iovec       vector[MAX_IOVEC];
-        int                count;
-        struct iovec      *pending_vector;
-        int                pending_count;
-        char              *buf;
-        struct iobref     *iobref;
-};
-
-
-typedef struct {
-        int32_t                sock;
-        int32_t                idx;
-        unsigned char          connected; // -1 = not connected. 0 = in progress. 1 = connected
-        char                   bio;
-        char                   connect_finish_log;
-        char                   submit_log;
-        union {
-                struct list_head     ioq;
-                struct {
-                        struct ioq        *ioq_next;
-                        struct ioq        *ioq_prev;
-                };
-        };
-        struct {
-                int                  state;
-                struct socket_header header;
-                char                *hdr_p;
-                size_t               hdrlen;
-                struct iobuf        *iobuf;
-                char                *buf_p;
-                size_t               buflen;
-                struct iovec         vector[2];
-                int                  count;
-                struct iovec        *pending_vector;
-                int                  pending_count;
-        } incoming;
-        pthread_mutex_t        lock;
-        int                    windowsize;
-        char                   lowlat;
-        char                   nodelay;
-        int                    keepalive;
-        int                    keepaliveintvl;
-} socket_private_t;
-
-
-#endif
diff --git a/xlators/protocol/server/src/Makefile.am b/xlators/protocol/server/src/Makefile.am
index 72a581ddf..6a18bf025 100644
--- a/xlators/protocol/server/src/Makefile.am
+++ b/xlators/protocol/server/src/Makefile.am
@@ -1,24 +1,26 @@
 xlator_LTLIBRARIES = server.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/protocol
 
-server_la_LDFLAGS = -module -avoidversion
+server_la_LDFLAGS = -module -avoid-version
 
 server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
 server_la_SOURCES = server.c server-resolve.c server-helpers.c  \
-	server3_1-fops.c server-handshake.c authenticate.c
+	server-rpc-fops.c server-handshake.c authenticate.c
 
 noinst_HEADERS = server.h server-helpers.h server-mem-types.h authenticate.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall    \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles  \
-	-DCONFDIR=\"$(sysconfdir)/glusterfs\" -D$(GF_HOST_OS)   \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src \
+	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
 	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
-	$(GF_CFLAGS) -I$(top_srcdir)/xlators/protocol/lib/src   \
-	-I$(top_srcdir)/rpc/rpc-lib/src/       \
-	-I$(top_srcdir)/rpc/xdr/src/       \
-	-I$(top_srcdir)/contrib/md5/
+	-I$(top_srcdir)/xlators/protocol/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS) \
+            -DDATADIR=\"$(localstatedir)\"
 
 CLEANFILES = *~
diff --git a/xlators/protocol/server/src/authenticate.c b/xlators/protocol/server/src/authenticate.c
index de5bf9f43..d8d138a84 100644
--- a/xlators/protocol/server/src/authenticate.c
+++ b/xlators/protocol/server/src/authenticate.c
@@ -1,21 +1,14 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
 
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -31,15 +24,15 @@
 #include <errno.h>
 #include "authenticate.h"
 
-static void
+static int
 init (dict_t *this, char *key, data_t *value, void *data)
 {
-        void *handle = NULL;
-        char *auth_file = NULL;
-        auth_handle_t *auth_handle = NULL;
-        auth_fn_t authenticate = NULL;
-        int *error = NULL;
-        int  ret = 0;
+        void          *handle       = NULL;
+        char          *auth_file    = NULL;
+        auth_handle_t *auth_handle  = NULL;
+        auth_fn_t      authenticate = NULL;
+        int           *error        = NULL;
+        int            ret          = 0;
 
         /* It gets over written */
         error = data;
@@ -59,7 +52,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
         if (-1 == ret) {
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 *error = -1;
-                return;
+                return -1;
         }
 
         handle = dlopen (auth_file, RTLD_LAZY);
@@ -69,7 +62,7 @@ init (dict_t *this, char *key, data_t *value, void *data)
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 GF_FREE (auth_file);
                 *error = -1;
-                return;
+                return -1;
         }
         GF_FREE (auth_file);
 
@@ -78,8 +71,9 @@ init (dict_t *this, char *key, data_t *value, void *data)
                 gf_log ("authenticate", GF_LOG_ERROR,
                         "dlsym(gf_auth) on %s\n", dlerror ());
                 dict_set (this, key, data_from_dynptr (NULL, 0));
+                dlclose (handle);
                 *error = -1;
-                return;
+                return -1;
         }
 
         auth_handle = GF_CALLOC (1, sizeof (*auth_handle),
@@ -87,10 +81,18 @@ init (dict_t *this, char *key, data_t *value, void *data)
         if (!auth_handle) {
                 dict_set (this, key, data_from_dynptr (NULL, 0));
                 *error = -1;
-                return;
+                dlclose (handle);
+                return -1;
         }
         auth_handle->vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
                                           gf_common_mt_volume_opt_list_t);
+        if (!auth_handle->vol_opt) {
+                dict_set (this, key, data_from_dynptr (NULL, 0));
+                *error = -1;
+                GF_FREE (auth_handle);
+                dlclose (handle);
+                return -1;
+        }
         auth_handle->vol_opt->given_opt = dlsym (handle, "options");
         if (auth_handle->vol_opt->given_opt == NULL) {
                 gf_log ("authenticate", GF_LOG_DEBUG,
@@ -102,47 +104,54 @@ init (dict_t *this, char *key, data_t *value, void *data)
 
         dict_set (this, key,
                   data_from_dynptr (auth_handle, sizeof (*auth_handle)));
+        return 0;
 }
 
-static void
+static int
 fini (dict_t *this, char *key, data_t *value, void *data)
 {
         auth_handle_t *handle = data_to_ptr (value);
         if (handle) {
                 dlclose (handle->handle);
         }
+        return 0;
+}
+
+static int
+_gf_auth_option_validate (dict_t *d, char *k, data_t *v, void *tmp)
+{
+        auth_handle_t *handle = NULL;
+        xlator_t      *xl = NULL;
+        int ret = 0;
+
+        xl = tmp;
+
+        handle = data_to_ptr (v);
+        if (!handle)
+                return 0;
+
+        list_add_tail (&(handle->vol_opt->list), &(xl->volume_options));
+
+        ret = xlator_options_validate_list (xl, xl->options,
+                                            handle->vol_opt, NULL);
+        if (ret) {
+                gf_log ("authenticate", GF_LOG_ERROR,
+                        "volume option validation failed");
+                return -1;
+        }
+        return 0;
 }
 
 int32_t
 gf_auth_init (xlator_t *xl, dict_t *auth_modules)
 {
         int ret = 0;
-        auth_handle_t *handle = NULL;
-        data_pair_t *pair = NULL;
 
         dict_foreach (auth_modules, init, &ret);
         if (ret)
                 goto out;
 
-        pair = auth_modules->members_list;
-        while (pair) {
-                handle = data_to_ptr (pair->value);
-                if (!handle) {
-                        pair = pair->next;
-                        continue;
-                }
-
-                list_add_tail (&(handle->vol_opt->list),
-                               &(xl->volume_options));
-                ret = xlator_options_validate_list (xl, xl->options,
-                                                    handle->vol_opt, NULL);
-
-                if (ret)
-                        gf_log ("authenticate", GF_LOG_ERROR,
-                                "volume option validation failed");
-
-                pair = pair->next;
-        }
+        ret = dict_foreach (auth_modules, _gf_auth_option_validate, xl);
 
 out:
         if (ret) {
@@ -156,7 +165,7 @@ out:
 static dict_t *__input_params;
 static dict_t *__config_params;
 
-void
+int
 map (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t *res = data;
@@ -171,15 +180,16 @@ map (dict_t *this, char *key, data_t *value, void *data)
         } else {
                 dict_set (res, key, int_to_data (AUTH_DONT_CARE));
         }
+        return 0;
 }
 
-void
+int
 reduce (dict_t *this, char *key, data_t *value, void *data)
 {
         int64_t val = 0;
         int64_t *res = data;
         if (!data)
-                return;
+                return 0;
 
         val = data_to_int64 (value);
         switch (val)
@@ -196,6 +206,7 @@ reduce (dict_t *this, char *key, data_t *value, void *data)
         case AUTH_DONT_CARE:
                 break;
         }
+        return 0;
 }
 
 
diff --git a/xlators/protocol/server/src/authenticate.h b/xlators/protocol/server/src/authenticate.h
index bb9f77e45..d4d43e498 100644
--- a/xlators/protocol/server/src/authenticate.h
+++ b/xlators/protocol/server/src/authenticate.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _AUTHENTICATE_H
diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index d1e3659ab..708acd936 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -40,7 +31,7 @@ gf_compare_client_version (rpcsvc_request_t *req, int fop_prognum,
 {
         int ret = -1;
         /* TODO: think.. */
-        if (glusterfs3_1_fop_prog.prognum == fop_prognum)
+        if (glusterfs3_3_fop_prog.prognum == fop_prognum)
                 ret = 0;
 
         return ret;
@@ -103,9 +94,9 @@ _volfile_update_checksum (xlator_t *this, char *key, uint32_t checksum)
 
         if (temp_volfile->checksum != checksum) {
                 gf_log (this->name, GF_LOG_INFO,
-                        "the volume file got modified between earlier access "
-                        "and now, this may lead to inconsistency between "
-                        "clients, advised to remount client");
+                        "the volume file was modified between a prior access "
+                        "and now. This may lead to inconsistency between "
+                        "clients, you are advised to remount client");
                 temp_volfile->checksum  = checksum;
         }
 
@@ -118,10 +109,10 @@ static size_t
 getspec_build_volfile_path (xlator_t *this, const char *key, char *path,
                             size_t path_len)
 {
-        int              ret = -1;
+        char            *filename      = NULL;
+        server_conf_t   *conf          = NULL;
+        int              ret           = -1;
         int              free_filename = 0;
-        char            *filename = NULL;
-        server_conf_t   *conf = NULL;
         char             data_key[256] = {0,};
 
         conf = this->private;
@@ -190,7 +181,7 @@ int
 _validate_volfile_checksum (xlator_t *this, char *key,
                             uint32_t checksum)
 {
-        char                 filename[ZR_PATH_MAX] = {0,};
+        char                 filename[PATH_MAX] = {0,};
         server_conf_t       *conf         = NULL;
         struct _volfile_ctx *temp_volfile = NULL;
         int                  ret          = 0;
@@ -253,7 +244,7 @@ server_getspec (rpcsvc_request_t *req)
         int32_t              op_errno               = ENOENT;
         int32_t              spec_fd                = -1;
         size_t               file_len               = 0;
-        char                 filename[ZR_PATH_MAX]  = {0,};
+        char                 filename[PATH_MAX]  = {0,};
         struct stat          stbuf                  = {0,};
         uint32_t             checksum               = 0;
         char                *key                    = NULL;
@@ -264,7 +255,9 @@ server_getspec (rpcsvc_request_t *req)
 
         this = req->svc->mydata;
         conf = this->private;
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gf_getspec_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_getspec_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 op_errno = EINVAL;
@@ -311,8 +304,6 @@ server_getspec (rpcsvc_request_t *req)
                         goto fail;
                 }
                 ret = read (spec_fd, rsp.spec, file_len);
-
-                close (spec_fd);
         }
 
         /* convert to XDR */
@@ -323,6 +314,9 @@ fail:
         rsp.op_errno = gf_errno_to_error (op_errno);
         rsp.op_ret   = ret;
 
+        if (spec_fd != -1)
+                close (spec_fd);
+
         server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
                              (xdrproc_t)xdr_gf_getspec_rsp);
 
@@ -335,14 +329,15 @@ server_setvolume (rpcsvc_request_t *req)
 {
         gf_setvolume_req     args          = {{0,},};
         gf_setvolume_rsp     rsp           = {0,};
-        server_connection_t *conn          = NULL;
+        client_t            *client        = NULL;
+        server_ctx_t        *serv_ctx      = NULL;
         server_conf_t       *conf          = NULL;
         peer_info_t         *peerinfo      = NULL;
         dict_t              *reply         = NULL;
         dict_t              *config_params = NULL;
         dict_t              *params        = NULL;
         char                *name          = NULL;
-        char                *process_uuid  = NULL;
+        char                *client_uid    = NULL;
         char                *clnt_version  = NULL;
         xlator_t            *xl            = NULL;
         char                *msg           = NULL;
@@ -360,7 +355,9 @@ server_setvolume (rpcsvc_request_t *req)
 
         params = dict_new ();
         reply  = dict_new ();
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gf_setvolume_req)) {
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_setvolume_req);
+        if (ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
@@ -397,7 +394,7 @@ server_setvolume (rpcsvc_request_t *req)
         params->extra_free = buf;
         buf = NULL;
 
-        ret = dict_get_str (params, "process-uuid", &process_uuid);
+        ret = dict_get_str (params, "process-uuid", &client_uid);
         if (ret < 0) {
                 ret = dict_set_str (reply, "ERROR",
                                     "UUID not specified");
@@ -414,7 +411,7 @@ server_setvolume (rpcsvc_request_t *req)
         ret = dict_get_uint32 (params, "clnt-lk-version", &lk_version);
         if (ret < 0) {
                 ret = dict_set_str (reply, "ERROR",
-                                    "lock state verison not supplied");
+                                    "lock state version not supplied");
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "failed to set error msg");
@@ -424,23 +421,34 @@ server_setvolume (rpcsvc_request_t *req)
                 goto fail;
         }
 
-        conn = server_connection_get (this, process_uuid);
-        if (!conn) {
+        client = gf_client_get (this, &req->cred, client_uid);
+        if (client == NULL) {
                 op_ret = -1;
                 op_errno = ENOMEM;
                 goto fail;
         }
 
-        cancelled = server_cancel_conn_timer (this, conn);
-        if (cancelled)
-                server_conn_unref (conn);
-        if (conn->lk_version != 0 &&
-            conn->lk_version != lk_version) {
-                (void) server_connection_cleanup (this, conn);
+        gf_log (this->name, GF_LOG_DEBUG, "Connected to %s", client->client_uid);
+        cancelled = server_cancel_grace_timer (this, client);
+        if (cancelled)//Do gf_client_put on behalf of grace-timer-handler.
+                gf_client_put (client, NULL);
+
+        serv_ctx = server_ctx_get (client, client->this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto fail;
         }
 
-        if (req->trans->xl_private != conn)
-                req->trans->xl_private = conn;
+        if (serv_ctx->lk_version != 0 &&
+            serv_ctx->lk_version != lk_version) {
+                (void) server_connection_cleanup (this, client,
+                                                  INTERNAL_LOCKS | POSIX_LOCKS);
+        }
+
+        if (req->trans->xl_private != client)
+                req->trans->xl_private = client;
+
+        auth_set_username_passwd (params, config_params, client);
 
         ret = dict_get_int32 (params, "fops-version", &fop_version);
         if (ret < 0) {
@@ -565,10 +573,10 @@ server_setvolume (rpcsvc_request_t *req)
 
                 gf_log (this->name, GF_LOG_INFO,
                         "accepted client from %s (version: %s)",
-                        (peerinfo) ? peerinfo->identifier : "<>",
+                        client->client_uid,
                         (clnt_version) ? clnt_version : "old");
                 op_ret = 0;
-                conn->bound_xl = xl;
+                client->bound_xl = xl;
                 ret = dict_set_str (reply, "ERROR", "Success");
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_DEBUG,
@@ -576,7 +584,7 @@ server_setvolume (rpcsvc_request_t *req)
         } else {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Cannot authenticate client from %s %s",
-                        (peerinfo) ? peerinfo->identifier : "<>",
+                        client->client_uid,
                         (clnt_version) ? clnt_version : "old");
 
                 op_ret = -1;
@@ -585,11 +593,10 @@ server_setvolume (rpcsvc_request_t *req)
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "failed to set error msg");
-
                 goto fail;
         }
 
-        if (conn->bound_xl == NULL) {
+        if (client->bound_xl == NULL) {
                 ret = dict_set_str (reply, "ERROR",
                                     "Check volfile and handshake "
                                     "options in protocol/client");
@@ -602,20 +609,21 @@ server_setvolume (rpcsvc_request_t *req)
                 goto fail;
         }
 
-        if ((conn->bound_xl != NULL) &&
+        if ((client->bound_xl != NULL) &&
             (ret >= 0)                   &&
-            (conn->bound_xl->itable == NULL)) {
+            (client->bound_xl->itable == NULL)) {
                 /* create inode table for this bound_xl, if one doesn't
                    already exist */
 
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating inode table with lru_limit=%"PRId32", "
                         "xlator=%s", conf->inode_lru_limit,
-                        conn->bound_xl->name);
+                        client->bound_xl->name);
 
                 /* TODO: what is this ? */
-                conn->bound_xl->itable = inode_table_new (conf->inode_lru_limit,
-                                                          conn->bound_xl);
+                client->bound_xl->itable =
+                        inode_table_new (conf->inode_lru_limit,
+                                         client->bound_xl);
         }
 
         ret = dict_set_str (reply, "process-uuid",
@@ -624,8 +632,7 @@ server_setvolume (rpcsvc_request_t *req)
                 gf_log (this->name, GF_LOG_DEBUG,
                         "failed to set 'process-uuid'");
 
-        ret = dict_set_uint32 (reply, "clnt-lk-version",
-                               conn->lk_version);
+        ret = dict_set_uint32 (reply, "clnt-lk-version", serv_ctx->lk_version);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
                         "failed to set 'clnt-lk-version'");
@@ -662,23 +669,34 @@ fail:
         rsp.op_ret   = op_ret;
         rsp.op_errno = gf_errno_to_error (op_errno);
 
+        /* if bound_xl is NULL or something fails, then put the connection
+         * back. Otherwise the connection would have been added to the
+         * list of connections the server is maintaining and might segfault
+         * during statedump when bound_xl of the connection is accessed.
+         */
+        if (op_ret && !xl) {
+                /* We would have set the xl_private of the transport to the
+                 * @conn. But if we have put the connection i.e shutting down
+                 * the connection, then we should set xl_private to NULL as it
+                 * would be pointing to a freed memory and would segfault when
+                 * accessed upon getting DISCONNECT.
+                 */
+                gf_client_put (client, NULL);
+                req->trans->xl_private = NULL;
+        }
         server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
                              (xdrproc_t)xdr_gf_setvolume_rsp);
 
 
-        if (args.dict.dict_val)
-                free (args.dict.dict_val);
+        free (args.dict.dict_val);
 
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
+        GF_FREE (rsp.dict.dict_val);
 
         dict_unref (params);
         dict_unref (reply);
         dict_unref (config_params);
 
-        if (buf) {
-                GF_FREE (buf);
-        }
+        GF_FREE (buf);
 
         return 0;
 }
@@ -701,12 +719,13 @@ server_ping (rpcsvc_request_t *req)
 int
 server_set_lk_version (rpcsvc_request_t *req)
 {
-        int                     op_ret          = -1;
-        int                     op_errno        = EINVAL;
-        gf_set_lk_ver_req       args            = {0, };
-        gf_set_lk_ver_rsp       rsp             = {0,};
-        server_connection_t     *conn           = NULL;
-        xlator_t                *this           = NULL;
+        int                 op_ret   = -1;
+        int                 op_errno = EINVAL;
+        gf_set_lk_ver_req   args     = {0,};
+        gf_set_lk_ver_rsp   rsp      = {0,};
+        client_t           *client   = NULL;
+        server_ctx_t       *serv_ctx = NULL;
+        xlator_t           *this     = NULL;
 
         this = req->svc->mydata;
         //TODO: Decide on an appropriate errno for the error-path
@@ -714,16 +733,23 @@ server_set_lk_version (rpcsvc_request_t *req)
         if (!this)
                 goto fail;
 
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_gf_set_lk_ver_req)) {
+        op_ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_set_lk_ver_req);
+        if (op_ret < 0) {
                 //failed to decode msg;
                 req->rpc_err = GARBAGE_ARGS;
                 goto fail;
         }
 
-        conn = server_connection_get (this, args.uid);
-        conn->lk_version = args.lk_ver;
-        server_conn_unref (conn);
+        client = gf_client_get (this, &req->cred, args.uid);
+        serv_ctx = server_ctx_get (client, client->this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto fail;
+        }
+
+        serv_ctx->lk_version = args.lk_ver;
+        gf_client_put (client, NULL);
 
         rsp.lk_ver   = args.lk_ver;
 
@@ -733,15 +759,18 @@ fail:
         rsp.op_errno = op_errno;
         server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
                              (xdrproc_t)xdr_gf_set_lk_ver_rsp);
+
+        free (args.uid);
+
         return 0;
 }
 
 rpcsvc_actor_t gluster_handshake_actors[] = {
-        [GF_HNDSK_NULL]       = {"NULL",      GF_HNDSK_NULL,      server_null, NULL, NULL, 0},
-        [GF_HNDSK_SETVOLUME]  = {"SETVOLUME", GF_HNDSK_SETVOLUME, server_setvolume, NULL, NULL, 0},
-        [GF_HNDSK_GETSPEC]    = {"GETSPEC",   GF_HNDSK_GETSPEC,   server_getspec, NULL, NULL, 0},
-        [GF_HNDSK_PING]       = {"PING",      GF_HNDSK_PING,      server_ping, NULL, NULL, 0},
-        [GF_HNDSK_SET_LK_VER] = {"SET_LK_VER", GF_HNDSK_SET_LK_VER, server_set_lk_version, NULL, NULL },
+        [GF_HNDSK_NULL]       = {"NULL",       GF_HNDSK_NULL,       server_null,           NULL, 0, DRC_NA},
+        [GF_HNDSK_SETVOLUME]  = {"SETVOLUME",  GF_HNDSK_SETVOLUME,  server_setvolume,      NULL, 0, DRC_NA},
+        [GF_HNDSK_GETSPEC]    = {"GETSPEC",    GF_HNDSK_GETSPEC,    server_getspec,        NULL, 0, DRC_NA},
+        [GF_HNDSK_PING]       = {"PING",       GF_HNDSK_PING,       server_ping,           NULL, 0, DRC_NA},
+        [GF_HNDSK_SET_LK_VER] = {"SET_LK_VER", GF_HNDSK_SET_LK_VER, server_set_lk_version, NULL, 0, DRC_NA},
 };
 
 
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 28d59aa44..c11011abf 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -35,6 +26,9 @@ server_decode_groups (call_frame_t *frame, rpcsvc_request_t *req)
         GF_VALIDATE_OR_GOTO ("server", frame, out);
         GF_VALIDATE_OR_GOTO ("server", req, out);
 
+        if (call_stack_alloc_groups (frame->root, req->auxgidcount) != 0)
+                return -1;
+
         frame->root->ngrps = req->auxgidcount;
         if (frame->root->ngrps == 0)
                 return 0;
@@ -48,6 +42,7 @@ out:
         return 0;
 }
 
+
 void
 server_loc_wipe (loc_t *loc)
 {
@@ -61,19 +56,16 @@ server_loc_wipe (loc_t *loc)
                 loc->inode = NULL;
         }
 
-        if (loc->path)
-                GF_FREE ((void *)loc->path);
+        GF_FREE ((void *)loc->path);
 }
 
 
 void
 server_resolve_wipe (server_resolve_t *resolve)
 {
-        if (resolve->path)
-                GF_FREE ((void *)resolve->path);
+        GF_FREE ((void *)resolve->path);
 
-        if (resolve->bname)
-                GF_FREE ((void *)resolve->bname);
+        GF_FREE ((void *)resolve->bname);
 
         loc_wipe (&resolve->resolve_loc);
 }
@@ -82,11 +74,6 @@ server_resolve_wipe (server_resolve_t *resolve)
 void
 free_state (server_state_t *state)
 {
-        if (state->conn) {
-                //xprt_svc_unref (state->conn);
-                state->conn = NULL;
-        }
-
         if (state->xprt) {
                 rpc_transport_unref (state->xprt);
                 state->xprt = NULL;
@@ -116,11 +103,14 @@ free_state (server_state_t *state)
                 state->dict = NULL;
         }
 
-        if (state->volume)
-                GF_FREE ((void *)state->volume);
+        if (state->xdata) {
+                dict_unref (state->xdata);
+                state->xdata = NULL;
+        }
 
-        if (state->name)
-                GF_FREE ((void *)state->name);
+        GF_FREE ((void *)state->volume);
+
+        GF_FREE ((void *)state->name);
 
         server_loc_wipe (&state->loc);
         server_loc_wipe (&state->loc2);
@@ -132,306 +122,26 @@ free_state (server_state_t *state)
 }
 
 
-int
-gf_add_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc, fd_t *fd, pid_t pid, gf_lkowner_t *owner,
-               glusterfs_fop_t type)
-{
-        int32_t         ret = -1;
-        struct _locker *new = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", table, out);
-        GF_VALIDATE_OR_GOTO ("server", volume, out);
-
-        new = GF_CALLOC (1, sizeof (struct _locker), gf_server_mt_locker_t);
-        if (new == NULL) {
-                goto out;
-        }
-        INIT_LIST_HEAD (&new->lockers);
-
-        new->volume = gf_strdup (volume);
-
-        if (fd == NULL) {
-                loc_copy (&new->loc, loc);
-        } else {
-                new->fd = fd_ref (fd);
-        }
-
-        new->pid   = pid;
-        new->owner = *owner;
-
-        LOCK (&table->lock);
-        {
-                if (type == GF_FOP_ENTRYLK)
-                        list_add_tail (&new->lockers, &table->entrylk_lockers);
-                else
-                        list_add_tail (&new->lockers, &table->inodelk_lockers);
-        }
-        UNLOCK (&table->lock);
-out:
-        return ret;
-}
-
-
-int
-gf_del_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc, fd_t *fd, gf_lkowner_t *owner,
-               glusterfs_fop_t type)
-{
-        struct _locker    *locker = NULL;
-        struct _locker    *tmp = NULL;
-        int32_t            ret = -1;
-        struct list_head  *head = NULL;
-        struct list_head   del;
-
-        GF_VALIDATE_OR_GOTO ("server", table, out);
-        GF_VALIDATE_OR_GOTO ("server", volume, out);
-
-        INIT_LIST_HEAD (&del);
-
-        LOCK (&table->lock);
-        {
-                if (type == GF_FOP_ENTRYLK) {
-                        head = &table->entrylk_lockers;
-                } else {
-                        head = &table->inodelk_lockers;
-                }
-
-                list_for_each_entry_safe (locker, tmp, head, lockers) {
-                        if (!is_same_lkowner (&locker->owner, owner) ||
-                            strcmp (locker->volume, volume))
-                                continue;
-
-                        if (locker->fd && fd && (locker->fd == fd))
-                                list_move_tail (&locker->lockers, &del);
-                        else if (locker->loc.inode && loc &&
-                                 (locker->loc.inode == loc->inode))
-                                list_move_tail (&locker->lockers, &del);
-                }
-        }
-        UNLOCK (&table->lock);
-
-        tmp = NULL;
-        locker = NULL;
-
-        list_for_each_entry_safe (locker, tmp, &del, lockers) {
-                list_del_init (&locker->lockers);
-                if (locker->fd)
-                        fd_unref (locker->fd);
-                else
-                        loc_wipe (&locker->loc);
-
-                GF_FREE (locker->volume);
-                GF_FREE (locker);
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-static struct _lock_table *
-gf_lock_table_new (void)
-{
-        struct _lock_table *new = NULL;
-
-        new = GF_CALLOC (1, sizeof (struct _lock_table), gf_server_mt_lock_table_t);
-        if (new == NULL) {
-                goto out;
-        }
-        INIT_LIST_HEAD (&new->entrylk_lockers);
-        INIT_LIST_HEAD (&new->inodelk_lockers);
-        LOCK_INIT (&new->lock);
-out:
-        return new;
-}
-
-static int
-server_nop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno)
-{
-        int             ret   = -1;
-        server_state_t *state = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
-        GF_VALIDATE_OR_GOTO ("server", cookie, out);
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-
-        if (frame->root->trans)
-                server_conn_unref (frame->root->trans);
-        state = CALL_STATE(frame);
-
-        if (state)
-                free_state (state);
-        STACK_DESTROY (frame->root);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int
-do_lock_table_cleanup (xlator_t *this, server_connection_t *conn,
-                       call_frame_t *frame, struct _lock_table *ltable)
-{
-        struct list_head  inodelk_lockers, entrylk_lockers;
-        call_frame_t     *tmp_frame = NULL;
-        struct gf_flock      flock = {0, };
-        xlator_t         *bound_xl = NULL;
-        struct _locker   *locker = NULL, *tmp = NULL;
-        int               ret = -1;
-        char             *path = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
-        GF_VALIDATE_OR_GOTO ("server", ltable, out);
-
-        bound_xl = conn->bound_xl;
-        INIT_LIST_HEAD (&inodelk_lockers);
-        INIT_LIST_HEAD (&entrylk_lockers);
-
-        LOCK (&ltable->lock);
-        {
-                list_splice_init (&ltable->inodelk_lockers,
-                                  &inodelk_lockers);
-
-                list_splice_init (&ltable->entrylk_lockers, &entrylk_lockers);
-        }
-        UNLOCK (&ltable->lock);
-
-        LOCK_DESTROY (&ltable->lock);
-        GF_FREE (ltable);
-
-        flock.l_type  = F_UNLCK;
-        flock.l_start = 0;
-        flock.l_len   = 0;
-        list_for_each_entry_safe (locker,
-                                  tmp, &inodelk_lockers, lockers) {
-                tmp_frame = copy_frame (frame);
-                if (tmp_frame == NULL) {
-                        goto out;
-                }
-                /*
-                  lock owner = 0 is a special case that tells posix-locks
-                  to release all locks from this transport
-                */
-                tmp_frame->root->pid         = 0;
-                tmp_frame->root->trans       = server_conn_ref (conn);
-                memset (&tmp_frame->root->lk_owner, 0, sizeof (gf_lkowner_t));
-
-                if (locker->fd) {
-                        GF_ASSERT (locker->fd->inode);
-
-                        ret = inode_path (locker->fd->inode, NULL, &path);
-
-                        if (ret > 0) {
-                                gf_log (this->name, GF_LOG_INFO, "finodelk "
-                                        "released on %s", path);
-                                GF_FREE (path);
-                        } else {
-
-                                gf_log (this->name, GF_LOG_INFO, "finodelk "
-                                        "released on inode with gfid %s",
-                                        uuid_utoa (locker->fd->inode->gfid));
-                        }
-
-                        STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                    bound_xl->fops->finodelk,
-                                    locker->volume,
-                                    locker->fd, F_SETLK, &flock);
-                        fd_unref (locker->fd);
-                } else {
-                        gf_log (this->name, GF_LOG_INFO, "inodelk released "
-                                "on %s", locker->loc.path);
-
-                        STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                    bound_xl->fops->inodelk,
-                                    locker->volume,
-                                    &(locker->loc), F_SETLK, &flock);
-                        loc_wipe (&locker->loc);
-                }
-
-                GF_FREE (locker->volume);
-
-                list_del_init (&locker->lockers);
-                GF_FREE (locker);
-        }
-
-        tmp = NULL;
-        locker = NULL;
-        list_for_each_entry_safe (locker, tmp, &entrylk_lockers, lockers) {
-                tmp_frame = copy_frame (frame);
-
-                tmp_frame->root->pid         = 0;
-                tmp_frame->root->trans       = server_conn_ref (conn);
-                memset (&tmp_frame->root->lk_owner, 0, sizeof (gf_lkowner_t));
-
-                if (locker->fd) {
-                        GF_ASSERT (locker->fd->inode);
-
-                        ret = inode_path (locker->fd->inode, NULL, &path);
-
-                        if (ret > 0) {
-                                gf_log (this->name, GF_LOG_INFO, "fentrylk "
-                                        "released on %s", path);
-                                GF_FREE (path);
-                        }  else {
-
-                                gf_log (this->name, GF_LOG_INFO, "fentrylk "
-                                        "released on inode with gfid %s",
-                                        uuid_utoa (locker->fd->inode->gfid));
-                        }
-
-                        STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                    bound_xl->fops->fentrylk,
-                                    locker->volume,
-                                    locker->fd, NULL,
-                                    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                        fd_unref (locker->fd);
-                } else {
-                        gf_log (this->name, GF_LOG_INFO, "entrylk released "
-                                "on %s", locker->loc.path);
-
-                        STACK_WIND (tmp_frame, server_nop_cbk, bound_xl,
-                                    bound_xl->fops->entrylk,
-                                    locker->volume,
-                                    &(locker->loc), NULL,
-                                    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-                        loc_wipe (&locker->loc);
-                }
-
-                GF_FREE (locker->volume);
-
-                list_del_init (&locker->lockers);
-                GF_FREE (locker);
-        }
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
 static int
 server_connection_cleanup_flush_cbk (call_frame_t *frame, void *cookie,
                                      xlator_t *this, int32_t op_ret,
-                                     int32_t op_errno)
+                                     int32_t op_errno, dict_t *xdata)
 {
-        int32_t ret = -1;
-        fd_t *fd = NULL;
+        int32_t    ret    = -1;
+        fd_t      *fd     = NULL;
+        client_t  *client = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", this, out);
         GF_VALIDATE_OR_GOTO ("server", cookie, out);
         GF_VALIDATE_OR_GOTO ("server", frame, out);
 
         fd = frame->local;
+        client = frame->root->client;
 
         fd_unref (fd);
         frame->local = NULL;
 
-        if (frame->root->trans)
-                server_conn_unref (frame->root->trans);
+        gf_client_unref (client);
         STACK_DESTROY (frame->root);
 
         ret = 0;
@@ -440,9 +150,8 @@ out:
 }
 
 
-int
-do_fd_cleanup (xlator_t *this, server_connection_t *conn, call_frame_t *frame,
-               fdentry_t *fdentries, int fd_count)
+static int
+do_fd_cleanup (xlator_t *this, client_t* client, fdentry_t *fdentries, int fd_count)
 {
         fd_t               *fd = NULL;
         int                 i = 0, ret = -1;
@@ -451,16 +160,14 @@ do_fd_cleanup (xlator_t *this, server_connection_t *conn, call_frame_t *frame,
         char               *path     = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
         GF_VALIDATE_OR_GOTO ("server", fdentries, out);
 
-        bound_xl = conn->bound_xl;
+        bound_xl = client->bound_xl;
         for (i = 0;i < fd_count; i++) {
                 fd = fdentries[i].fd;
 
                 if (fd != NULL) {
-                        tmp_frame = copy_frame (frame);
+                        tmp_frame = create_frame (this, this->ctx->pool);
                         if (tmp_frame == NULL) {
                                 goto out;
                         }
@@ -470,26 +177,26 @@ do_fd_cleanup (xlator_t *this, server_connection_t *conn, call_frame_t *frame,
                         ret = inode_path (fd->inode, NULL, &path);
 
                         if (ret > 0) {
-                                gf_log (this->name, GF_LOG_INFO, "fd cleanup on "
-                                        "%s", path);
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "fd cleanup on %s", path);
                                 GF_FREE (path);
                         }  else {
 
-                                gf_log (this->name, GF_LOG_INFO, "fd cleanup on"
-                                        " inode with gfid %s",
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "fd cleanup on inode with gfid %s",
                                         uuid_utoa (fd->inode->gfid));
                         }
 
                         tmp_frame->local = fd;
 
                         tmp_frame->root->pid = 0;
-                        tmp_frame->root->trans = server_conn_ref (conn);
+                        gf_client_ref (client);
                         memset (&tmp_frame->root->lk_owner, 0,
                                 sizeof (gf_lkowner_t));
 
                         STACK_WIND (tmp_frame,
                                     server_connection_cleanup_flush_cbk,
-                                    bound_xl, bound_xl->fops->flush, fd);
+                                    bound_xl, bound_xl->fops->flush, fd, NULL);
                 }
         }
 
@@ -500,241 +207,72 @@ out:
         return ret;
 }
 
-int
-do_connection_cleanup (xlator_t *this, server_connection_t *conn,
-                       struct _lock_table *ltable, fdentry_t *fdentries, int fd_count)
-{
-        int             ret = 0;
-        int             saved_ret = 0;
-        call_frame_t   *frame = NULL;
-        server_state_t *state = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-        GF_VALIDATE_OR_GOTO ("server", fdentries, out);
-        GF_VALIDATE_OR_GOTO ("server", ltable, out);
-
-        frame = create_frame (this, this->ctx->pool);
-        if (frame == NULL) {
-                goto out;
-        }
-
-        saved_ret = do_lock_table_cleanup (this, conn, frame, ltable);
-
-        if (fdentries != NULL) {
-                ret = do_fd_cleanup (this, conn, frame, fdentries, fd_count);
-        }
-
-        state = CALL_STATE (frame);
-        if (state)
-                GF_FREE (state);
-
-        STACK_DESTROY (frame->root);
-
-        if (saved_ret || ret) {
-                ret = -1;
-        }
-
-out:
-        return ret;
-}
-
 
 int
-server_connection_cleanup (xlator_t *this, server_connection_t *conn)
+server_connection_cleanup (xlator_t *this, client_t *client,
+                           int32_t flags)
 {
-        struct _lock_table *ltable = NULL;
-        fdentry_t          *fdentries = NULL;
-        uint32_t            fd_count = 0;
-        int                 ret = 0;
+        server_ctx_t        *serv_ctx  = NULL;
+        fdentry_t           *fdentries = NULL;
+        uint32_t             fd_count  = 0;
+        int                  cd_ret    = 0;
+        int                  ret       = 0;
 
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (conn->ltable) {
-                        ltable = conn->ltable;
-                        conn->ltable = gf_lock_table_new ();
-                }
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, client, out);
+        GF_VALIDATE_OR_GOTO (this->name, flags, out);
 
-                if (conn->fdtable)
-                        fdentries = gf_fd_fdtable_get_all_fds (conn->fdtable,
-                                                               &fd_count);
-        }
-        pthread_mutex_unlock (&conn->lock);
-
-        if (conn->bound_xl)
-                ret = do_connection_cleanup (this, conn, ltable,
-                                             fdentries, fd_count);
-
-out:
-        return ret;
-}
+        serv_ctx = server_ctx_get (client, client->this);
 
-
-int
-server_connection_destroy (xlator_t *this, server_connection_t *conn)
-{
-        xlator_t           *bound_xl = NULL;
-        int32_t             ret = -1;
-        struct list_head    inodelk_lockers;
-        struct list_head    entrylk_lockers;
-        struct _lock_table *ltable = NULL;
-        fdtable_t          *fdtable = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-
-        bound_xl = (xlator_t *) (conn->bound_xl);
-
-        if (bound_xl) {
-                pthread_mutex_lock (&(conn->lock));
-                {
-                        if (conn->ltable) {
-                                ltable = conn->ltable;
-                                conn->ltable = NULL;
-                        }
-                        if (conn->fdtable) {
-                                fdtable = conn->fdtable;
-                                conn->fdtable = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conn->lock);
-
-                INIT_LIST_HEAD (&inodelk_lockers);
-                INIT_LIST_HEAD (&entrylk_lockers);
-
-                if (ltable) {
-                        LOCK (&ltable->lock);
-                        {
-                                list_splice_init (&ltable->inodelk_lockers,
-                                                  &inodelk_lockers);
-
-                                list_splice_init (&ltable->entrylk_lockers,
-                                                  &entrylk_lockers);
-                        }
-                        UNLOCK (&ltable->lock);
-                        LOCK_DESTROY (&ltable->lock);
-                        GF_FREE (ltable);
-                }
-
-                GF_ASSERT (list_empty (&inodelk_lockers));
-                GF_ASSERT (list_empty (&entrylk_lockers));
-
-                if (fdtable)
-                        gf_fd_fdtable_destroy (fdtable);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
         }
 
-        gf_log (this->name, GF_LOG_INFO, "destroyed connection of %s",
-                conn->id);
-
-        pthread_mutex_destroy (&conn->lock);
-        GF_FREE (conn->id);
-        GF_FREE (conn);
-        ret = 0;
-out:
-        return ret;
-}
-
-server_connection_t*
-server_conn_unref (server_connection_t *conn)
-{
-        server_connection_t *todel = NULL;
-        xlator_t            *this = NULL;
-
-        pthread_mutex_lock (&conn->lock);
+        LOCK (&serv_ctx->fdtable_lock);
         {
-                conn->ref--;
-
-                if (!conn->ref) {
-                        list_del_init (&conn->list);
-                        todel = conn;
-                }
+                if (serv_ctx->fdtable && (flags & POSIX_LOCKS))
+                        fdentries = gf_fd_fdtable_get_all_fds (serv_ctx->fdtable,
+                                                               &fd_count);
         }
-        pthread_mutex_unlock (&conn->lock);
+        UNLOCK (&serv_ctx->fdtable_lock);
 
-        if (todel) {
-                this = THIS;
-                server_connection_destroy (this, todel);
-                conn = NULL;
-        }
-        return conn;
-}
+        if (client->bound_xl == NULL)
+                goto out;
 
-server_connection_t*
-server_conn_ref (server_connection_t *conn)
-{
-        pthread_mutex_lock (&conn->lock);
-        {
-                conn->ref++;
+        if (flags & INTERNAL_LOCKS) {
+                cd_ret = gf_client_disconnect (client);
         }
-        pthread_mutex_unlock (&conn->lock);
-
-        return conn;
-}
 
-server_connection_t *
-server_connection_get (xlator_t *this, const char *id)
-{
-        server_connection_t *conn = NULL;
-        server_connection_t *trav = NULL;
-        server_conf_t       *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", id, out);
-
-        conf = this->private;
+        if (fdentries != NULL)
+                ret = do_fd_cleanup (this, client, fdentries, fd_count);
+        else
+                gf_log (this->name, GF_LOG_INFO, "no fdentries to clean");
 
-        pthread_mutex_lock (&conf->mutex);
-        {
-                list_for_each_entry (trav, &conf->conns, list) {
-                        if (!strncmp (trav->id, id, strlen (id))) {
-                                conn = trav;
-                                goto unlock;
-                        }
-                }
-
-                conn = (void *) GF_CALLOC (1, sizeof (*conn),
-                                           gf_server_mt_conn_t);
-                if (!conn)
-                        goto unlock;
-
-                conn->id = gf_strdup (id);
-                /*'0' denotes uninitialised lock state*/
-                conn->lk_version = 0;
-                conn->fdtable = gf_fd_fdtable_alloc ();
-                conn->ltable  = gf_lock_table_new ();
-                conn->this    = this;
-                pthread_mutex_init (&conn->lock, NULL);
-                list_add (&conn->list, &conf->conns);
+        if (cd_ret || ret)
+                ret = -1;
 
-        }
-unlock:
-        pthread_mutex_unlock (&conf->mutex);
 out:
-        if (conn)
-                server_conn_ref (conn);
-        return conn;
+        return ret;
 }
 
 
 static call_frame_t *
 server_alloc_frame (rpcsvc_request_t *req)
 {
-        call_frame_t         *frame = NULL;
-        server_state_t       *state = NULL;
-        server_connection_t  *conn  = NULL;
+        call_frame_t    *frame  = NULL;
+        server_state_t  *state  = NULL;
+        client_t        *client = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", req, out);
         GF_VALIDATE_OR_GOTO ("server", req->trans, out);
         GF_VALIDATE_OR_GOTO ("server", req->svc, out);
         GF_VALIDATE_OR_GOTO ("server", req->svc->ctx, out);
 
-        conn = (server_connection_t *)req->trans->xl_private;
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
+        client = req->trans->xl_private;
+        GF_VALIDATE_OR_GOTO ("server", client, out);
 
-        frame = create_frame (conn->this, req->svc->ctx->pool);
+        frame = create_frame (client->this, req->svc->ctx->pool);
         if (!frame)
                 goto out;
 
@@ -742,45 +280,102 @@ server_alloc_frame (rpcsvc_request_t *req)
         if (!state)
                 goto out;
 
-        if (conn->bound_xl)
-                state->itable = conn->bound_xl->itable;
+        if (client->bound_xl)
+                state->itable = client->bound_xl->itable;
 
         state->xprt  = rpc_transport_ref (req->trans);
-        state->conn  = conn;
-
         state->resolve.fd_no = -1;
         state->resolve2.fd_no = -1;
 
+        frame->root->client  = client;
         frame->root->state = state;        /* which socket */
         frame->root->unique = 0;           /* which call */
 
-        frame->this = conn->this;
+        frame->this = client->this;
 out:
         return frame;
 }
 
 
-
 call_frame_t *
 get_frame_from_request (rpcsvc_request_t *req)
 {
-        call_frame_t *frame = NULL;
+        call_frame_t  *frame = NULL;
+        client_t      *client = NULL;
+        client_t      *tmp_client = NULL;
+        xlator_t  *this = NULL;
+        server_conf_t *priv = NULL;
+        clienttable_t *clienttable = NULL;
+        unsigned int   i           = 0;
 
         GF_VALIDATE_OR_GOTO ("server", req, out);
 
+        client = req->trans->xl_private;
+
         frame = server_alloc_frame (req);
         if (!frame)
                 goto out;
 
         frame->root->op       = req->procnum;
-        frame->root->type     = req->type;
 
         frame->root->unique   = req->xid;
 
+        client = req->trans->xl_private;
+        this = req->trans->xl;
+        priv = this->private;
+        clienttable = this->ctx->clienttable;
+
+        for (i = 0; i < clienttable->max_clients; i++) {
+                tmp_client = clienttable->cliententries[i].client;
+                if (client == tmp_client) {
+                        /* for non trusted clients username and password
+                           would not have been set. So for non trusted clients
+                           (i.e clients not from the same machine as the brick,
+                           and clients from outside the storage pool)
+                           do the root-squashing.
+                           TODO: If any client within the storage pool (i.e
+                           mounting within a machine from the pool but using
+                           other machine's ip/hostname from the same pool)
+                           is present treat it as a trusted client
+                        */
+                        if (!client->auth.username && req->pid != NFS_PID)
+                                RPC_AUTH_ROOT_SQUASH (req);
+
+                        /* Problem: If we just check whether the client is
+                           trusted client and do not do root squashing for
+                           them, then for smb clients and UFO clients root
+                           squashing will never happen as they use the fuse
+                           mounts done within the trusted pool (i.e they are
+                           trusted clients).
+                           Solution: To fix it, do root squashing for trusted
+                           clients also. If one wants to have a client within
+                           the storage pool for which root-squashing does not
+                           happen, then the client has to be mounted with
+                           --no-root-squash option. But for defrag client and
+                           gsyncd client do not do root-squashing.
+                        */
+                        if (client->auth.username &&
+                            req->pid != GF_CLIENT_PID_NO_ROOT_SQUASH &&
+                            req->pid != GF_CLIENT_PID_GSYNCD &&
+                            req->pid != GF_CLIENT_PID_DEFRAG)
+                                RPC_AUTH_ROOT_SQUASH (req);
+
+                        /* For nfs clients the server processes will be running
+                           within the trusted storage pool machines. So if we
+                           do not do root-squashing for nfs servers, thinking
+                           that its a trusted client, then root-squashing wont
+                           work for nfs clients.
+                        */
+                        if (req->pid == NFS_PID)
+                                RPC_AUTH_ROOT_SQUASH (req);
+                }
+        }
+
         frame->root->uid      = req->uid;
         frame->root->gid      = req->gid;
         frame->root->pid      = req->pid;
-        frame->root->trans    = server_conn_ref (req->trans->xl_private);
+        gf_client_ref (client);
+        frame->root->client   = client;
         frame->root->lk_owner = req->lk_owner;
 
         server_decode_groups (frame, req);
@@ -865,84 +460,6 @@ out:
         return ret;
 }
 
-void
-put_server_conn_state (xlator_t *this, rpc_transport_t *xprt)
-{
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", xprt, out);
-
-        xprt->xl_private = NULL;
-out:
-        return;
-}
-
-server_connection_t *
-get_server_conn_state (xlator_t *this, rpc_transport_t *xprt)
-{
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", xprt, out);
-
-        return (server_connection_t *)xprt->xl_private;
-out:
-        return NULL;
-}
-
-server_connection_t *
-create_server_conn_state (xlator_t *this, rpc_transport_t *xprt)
-{
-        server_connection_t *conn = NULL;
-        int                  ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", xprt, out);
-
-        conn = GF_CALLOC (1, sizeof (*conn), gf_server_mt_conn_t);
-        if (!conn)
-                goto out;
-
-        pthread_mutex_init (&conn->lock, NULL);
-
-        conn->fdtable = gf_fd_fdtable_alloc ();
-        if (!conn->fdtable)
-                goto out;
-
-        conn->ltable  = gf_lock_table_new ();
-        if (!conn->ltable)
-                goto out;
-
-        conn->this = this;
-
-        xprt->xl_private = conn;
-
-        ret = 0;
-out:
-        if (ret)
-                destroy_server_conn_state (conn);
-
-        return conn;
-}
-
-void
-destroy_server_conn_state (server_connection_t *conn)
-{
-        GF_VALIDATE_OR_GOTO ("server", conn, out);
-
-        if (conn->ltable) {
-                /* TODO */
-                //FREE (conn->ltable);
-                ;
-        }
-
-        if (conn->fdtable)
-                gf_fd_fdtable_destroy (conn->fdtable);
-
-        pthread_mutex_destroy (&conn->lock);
-
-        GF_FREE (conn);
-out:
-        return;
-}
-
 
 void
 print_caller (char *str, int size, call_frame_t *frame)
@@ -1069,12 +586,15 @@ server_print_params (char *str, int size, server_state_t *state)
                 filled += snprintf (str + filled, size - filled,
                                     "volume=%s,", state->volume);
 
+/* FIXME
         snprintf (str + filled, size - filled,
-                  "bound_xl=%s}", state->conn->bound_xl->name);
+                  "bound_xl=%s}", state->client->bound_xl->name);
+*/
 out:
         return;
 }
 
+
 int
 server_resolve_is_empty (server_resolve_t *resolve)
 {
@@ -1090,6 +610,7 @@ server_resolve_is_empty (server_resolve_t *resolve)
         return 1;
 }
 
+
 void
 server_print_reply (call_frame_t *frame, int op_ret, int op_errno)
 {
@@ -1114,10 +635,7 @@ server_print_reply (call_frame_t *frame, int op_ret, int op_errno)
 
         switch (frame->root->type) {
         case GF_OP_TYPE_FOP:
-                op = gf_fop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_MGMT:
-                op = gf_mgmt_list[frame->root->op];
+                op = (char *)gf_fop_list[frame->root->op];
                 break;
         default:
                 op = "";
@@ -1138,16 +656,16 @@ out:
 void
 server_print_request (call_frame_t *frame)
 {
-        server_conf_t   *conf = NULL;
-        xlator_t        *this = NULL;
+        server_conf_t   *conf  = NULL;
+        xlator_t        *this  = NULL;
         server_state_t  *state = NULL;
+        char            *op    = "UNKNOWN";
         char             resolve_vars[256];
         char             resolve2_vars[256];
         char             loc_vars[256];
         char             loc2_vars[256];
         char             other_vars[512];
         char             caller[512];
-        char            *op = "UNKNOWN";
 
         GF_VALIDATE_OR_GOTO ("server", frame, out);
 
@@ -1183,10 +701,7 @@ server_print_request (call_frame_t *frame)
 
         switch (frame->root->type) {
         case GF_OP_TYPE_FOP:
-                op = gf_fop_list[frame->root->op];
-                break;
-        case GF_OP_TYPE_MGMT:
-                op = gf_mgmt_list[frame->root->op];
+                op = (char *)gf_fop_list[frame->root->op];
                 break;
         default:
                 op = "";
@@ -1201,13 +716,14 @@ out:
         return;
 }
 
+
 int
 serialize_rsp_direntp (gf_dirent_t *entries, gfs3_readdirp_rsp *rsp)
 {
         gf_dirent_t         *entry = NULL;
-        gfs3_dirplist       *trav = NULL;
-        gfs3_dirplist       *prev = NULL;
-        int                  ret = -1;
+        gfs3_dirplist       *trav  = NULL;
+        gfs3_dirplist       *prev  = NULL;
+        int                  ret   = -1;
 
         GF_VALIDATE_OR_GOTO ("server", entries, out);
         GF_VALIDATE_OR_GOTO ("server", rsp, out);
@@ -1261,10 +777,13 @@ serialize_rsp_direntp (gf_dirent_t *entries, gfs3_readdirp_rsp *rsp)
                         rsp->reply = trav;
 
                 prev = trav;
+                trav = NULL;
         }
 
         ret = 0;
 out:
+        GF_FREE (trav);
+
         return ret;
 }
 
@@ -1272,10 +791,10 @@ out:
 int
 serialize_rsp_dirent (gf_dirent_t *entries, gfs3_readdir_rsp *rsp)
 {
-        gf_dirent_t         *entry = NULL;
-        gfs3_dirlist        *trav = NULL;
-        gfs3_dirlist        *prev = NULL;
-        int                  ret = -1;
+        gf_dirent_t   *entry = NULL;
+        gfs3_dirlist  *trav  = NULL;
+        gfs3_dirlist  *prev  = NULL;
+        int           ret    = -1;
 
         GF_VALIDATE_OR_GOTO ("server", entries, out);
         GF_VALIDATE_OR_GOTO ("server", rsp, out);
@@ -1302,11 +821,12 @@ out:
         return ret;
 }
 
+
 int
 readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
 {
-        gfs3_dirlist *prev = NULL;
-        gfs3_dirlist *trav = NULL;
+        gfs3_dirlist  *prev = NULL;
+        gfs3_dirlist  *trav = NULL;
 
         trav = rsp->reply;
         prev = trav;
@@ -1319,6 +839,7 @@ readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
         return 0;
 }
 
+
 int
 readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
 {
@@ -1329,8 +850,7 @@ readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
         prev = trav;
         while (trav) {
                 trav = trav->nextentry;
-                if (prev->dict.dict_val)
-                        GF_FREE (prev->dict.dict_val);
+                GF_FREE (prev->dict.dict_val);
                 GF_FREE (prev);
                 prev = trav;
         }
@@ -1338,6 +858,7 @@ readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
         return 0;
 }
 
+
 int
 gf_server_check_getxattr_cmd (call_frame_t *frame, const char *key)
 {
@@ -1351,10 +872,14 @@ gf_server_check_getxattr_cmd (call_frame_t *frame, const char *key)
 
         if (fnmatch ("*list*mount*point*", key, 0) == 0) {
                 /* list all the client protocol connecting to this process */
-                list_for_each_entry (xprt, &conf->xprt_list, list) {
-                        gf_log ("mount-point-list", GF_LOG_INFO,
-                                "%s", xprt->peerinfo.identifier);
+                pthread_mutex_lock (&conf->mutex);
+                {
+                        list_for_each_entry (xprt, &conf->xprt_list, list) {
+                                gf_log ("mount-point-list", GF_LOG_INFO,
+                                        "%s", xprt->peerinfo.identifier);
+                        }
                 }
+                pthread_mutex_unlock (&conf->mutex);
         }
 
         /* Add more options/keys here */
@@ -1362,64 +887,208 @@ gf_server_check_getxattr_cmd (call_frame_t *frame, const char *key)
         return 0;
 }
 
+
 int
 gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict)
 {
 
-        data_pair_t      *pair = NULL;
-        server_conf_t    *conf = NULL;
-        rpc_transport_t  *xprt = NULL;
-        uint64_t          total_read = 0;
+        server_conf_t    *conf        = NULL;
+        rpc_transport_t  *xprt        = NULL;
+        uint64_t          total_read  = 0;
         uint64_t          total_write = 0;
 
         conf = frame->this->private;
-        if (!conf)
+        if (!conf || !dict)
                 return 0;
 
-        for (pair = dict->members_list; pair; pair = pair->next) {
-                /* this exact key is used in 'io-stats' too.
-                 * But this is better place for this information dump.
-                 */
-                if (fnmatch ("*io*stat*dump", pair->key, 0) == 0) {
-                        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                                total_read  += xprt->total_bytes_read;
-                                total_write += xprt->total_bytes_write;
-                        }
-                        gf_log ("stats", GF_LOG_INFO,
-                                "total-read %"PRIu64", total-write %"PRIu64,
-                                total_read, total_write);
+        if (dict_foreach_fnmatch (dict, "*io*stat*dump",
+                                  dict_null_foreach_fn, NULL ) > 0) {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        total_read  += xprt->total_bytes_read;
+                        total_write += xprt->total_bytes_write;
                 }
+                gf_log ("stats", GF_LOG_INFO,
+                        "total-read %"PRIu64", total-write %"PRIu64,
+                        total_read, total_write);
         }
 
         return 0;
 }
 
+
 gf_boolean_t
-server_cancel_conn_timer (xlator_t *this, server_connection_t *conn)
+server_cancel_grace_timer (xlator_t *this, client_t *client)
 {
-        gf_timer_t      *timer = NULL;
-        gf_boolean_t    cancelled = _gf_false;
+        server_ctx_t  *serv_ctx  = NULL;
+        gf_timer_t    *timer     = NULL;
+        gf_boolean_t   cancelled = _gf_false;
 
-        if (!this || !conn) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Invalid arguments to "
-                        "cancel connection timer");
+        if (!this || !client) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Invalid arguments to cancel connection timer");
                 return cancelled;
         }
 
-        pthread_mutex_lock (&conn->lock);
-        {
-                if (!conn->timer)
-                        goto unlock;
+        serv_ctx = server_ctx_get (client, client->this);
 
-                timer = conn->timer;
-                conn->timer = NULL;
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
         }
-unlock:
-        pthread_mutex_unlock (&conn->lock);
+
+        LOCK (&serv_ctx->fdtable_lock);
+        {
+                if (serv_ctx->grace_timer) {
+                        timer = serv_ctx->grace_timer;
+                        serv_ctx->grace_timer = NULL;
+                }
+        }
+        UNLOCK (&serv_ctx->fdtable_lock);
 
         if (timer) {
                 gf_timer_call_cancel (this->ctx, timer);
                 cancelled = _gf_true;
         }
+out:
         return cancelled;
 }
+
+server_ctx_t*
+server_ctx_get (client_t *client, xlator_t *xlator)
+{
+        void *tmp = NULL;
+        server_ctx_t *ctx = NULL;
+
+        client_ctx_get (client, xlator, &tmp);
+
+        ctx = tmp;
+
+        if (ctx != NULL)
+                goto out;
+
+        ctx = GF_CALLOC (1, sizeof (server_ctx_t), gf_server_mt_server_conf_t);
+
+        if (ctx == NULL)
+                goto out;
+
+     /* ctx->lk_version = 0; redundant */
+        ctx->fdtable = gf_fd_fdtable_alloc ();
+
+        if (ctx->fdtable == NULL) {
+                GF_FREE (ctx);
+                ctx = NULL;
+                goto out;
+        }
+
+        LOCK_INIT (&ctx->fdtable_lock);
+
+        if (client_ctx_set (client, xlator, ctx) != 0) {
+              LOCK_DESTROY (&ctx->fdtable_lock);
+              GF_FREE (ctx);
+              ctx = NULL;
+        }
+
+out:
+        return ctx;
+}
+
+int
+auth_set_username_passwd (dict_t *input_params, dict_t *config_params,
+                          client_t *client)
+{
+        int      ret           = 0;
+        data_t  *allow_user    = NULL;
+        data_t  *passwd_data   = NULL;
+        char    *username      = NULL;
+        char    *password      = NULL;
+        char    *brick_name    = NULL;
+        char    *searchstr     = NULL;
+        char    *username_str  = NULL;
+        char    *tmp           = NULL;
+        char    *username_cpy  = NULL;
+
+        ret = dict_get_str (input_params, "username", &username);
+        if (ret) {
+                gf_log ("auth/login", GF_LOG_DEBUG,
+                        "username not found, returning DONT-CARE");
+                /* For non trusted clients username and password
+                   will not be there. So dont reject the client.
+                */
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_str (input_params, "password", &password);
+        if (ret) {
+                gf_log ("auth/login", GF_LOG_WARNING,
+                        "password not found, returning DONT-CARE");
+                goto out;
+        }
+
+        ret = dict_get_str (input_params, "remote-subvolume", &brick_name);
+        if (ret) {
+                gf_log ("auth/login", GF_LOG_ERROR,
+                        "remote-subvolume not specified");
+                ret = -1;
+                goto out;
+        }
+
+        ret = gf_asprintf (&searchstr, "auth.login.%s.allow", brick_name);
+        if (-1 == ret) {
+                ret = 0;
+                goto out;
+        }
+
+        allow_user = dict_get (config_params, searchstr);
+        GF_FREE (searchstr);
+
+        if (allow_user) {
+                username_cpy = gf_strdup (allow_user->data);
+                if (!username_cpy)
+                        goto out;
+
+                username_str = strtok_r (username_cpy, " ,", &tmp);
+
+                while (username_str) {
+                        if (!fnmatch (username_str, username, 0)) {
+                                ret = gf_asprintf (&searchstr,
+                                                   "auth.login.%s.password",
+                                                   username);
+                                if (-1 == ret)
+                                        goto out;
+
+                                passwd_data = dict_get (config_params,
+                                                        searchstr);
+                                GF_FREE (searchstr);
+
+                                if (!passwd_data) {
+                                        gf_log ("auth/login", GF_LOG_ERROR,
+                                                "wrong username/password "
+                                                "combination");
+                                        ret = -1;
+                                        goto out;
+                                }
+
+                                ret = !((strcmp (data_to_str (passwd_data),
+                                                    password))?0: -1);
+                                if (!ret) {
+                                        client->auth.username =
+                                                gf_strdup (username);
+                                        client->auth.passwd =
+                                                gf_strdup (password);
+                                }
+                                if (ret == -1)
+                                        gf_log ("auth/login", GF_LOG_ERROR,
+                                                "wrong password for user %s",
+                                                username);
+                                break;
+                        }
+                        username_str = strtok_r (NULL, " ,", &tmp);
+                }
+        }
+
+out:
+        GF_FREE (username_cpy);
+
+        return ret;
+}
diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h
index 9a502b478..3c257b3bc 100644
--- a/xlators/protocol/server/src/server-helpers.h
+++ b/xlators/protocol/server/src/server-helpers.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _SERVER_HELPERS_H
@@ -24,13 +15,8 @@
 
 #define CALL_STATE(frame)   ((server_state_t *)frame->root->state)
 
-#define BOUND_XL(frame)     ((xlator_t *) CALL_STATE(frame)->conn->bound_xl)
-
 #define XPRT_FROM_FRAME(frame) ((rpc_transport_t *) CALL_STATE(frame)->xprt)
 
-#define SERVER_CONNECTION(frame)                                \
-        ((server_connection_t *) CALL_STATE(frame)->conn)
-
 #define SERVER_CONF(frame)                                              \
         ((server_conf_t *)XPRT_FROM_FRAME(frame)->this->private)
 
@@ -47,41 +33,18 @@ void free_state (server_state_t *state);
 
 void server_loc_wipe (loc_t *loc);
 
-int32_t
-gf_add_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc,
-               fd_t *fd,
-               pid_t pid,
-               gf_lkowner_t *owner,
-               glusterfs_fop_t type);
-
-int32_t
-gf_del_locker (struct _lock_table *table, const char *volume,
-               loc_t *loc,
-               fd_t *fd,
-               gf_lkowner_t *owner,
-               glusterfs_fop_t type);
-
 void
 server_print_request (call_frame_t *frame);
 
 call_frame_t *
 get_frame_from_request (rpcsvc_request_t *req);
 
-gf_boolean_t
-server_cancel_conn_timer (xlator_t *this, server_connection_t *conn);
-
-void
-put_server_conn_state (xlator_t *this, rpc_transport_t *xprt);
-
-server_connection_t *
-get_server_conn_state (xlator_t *this, rpc_transport_t *xptr);
-
-server_connection_t *
-create_server_conn_state (xlator_t *this, rpc_transport_t *xptr);
+int
+server_connection_cleanup (xlator_t *this, struct _client_t *client,
+                           int32_t flags);
 
-void
-destroy_server_conn_state (server_connection_t *conn);
+gf_boolean_t
+server_cancel_grace_timer (xlator_t *this, struct _client_t *client);
 
 int
 server_build_config (xlator_t *this, server_conf_t *conf);
@@ -90,5 +53,9 @@ int serialize_rsp_dirent (gf_dirent_t *entries, gfs3_readdir_rsp *rsp);
 int serialize_rsp_direntp (gf_dirent_t *entries, gfs3_readdirp_rsp *rsp);
 int readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp);
 int readdir_rsp_cleanup (gfs3_readdir_rsp *rsp);
+int auth_set_username_passwd (dict_t *input_params, dict_t *config_params,
+                              struct _client_t *client);
+
+server_ctx_t *server_ctx_get (client_t *client, xlator_t *xlator);
 
 #endif /* !_SERVER_HELPERS_H */
diff --git a/xlators/protocol/server/src/server-mem-types.h b/xlators/protocol/server/src/server-mem-types.h
index 5438ed6db..19c3466d3 100644
--- a/xlators/protocol/server/src/server-mem-types.h
+++ b/xlators/protocol/server/src/server-mem-types.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
diff --git a/xlators/protocol/server/src/server-resolve.c b/xlators/protocol/server/src/server-resolve.c
index 38ab368c3..b2bff5c53 100644
--- a/xlators/protocol/server/src/server-resolve.c
+++ b/xlators/protocol/server/src/server-resolve.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -70,7 +61,8 @@ resolve_loc_touchup (call_frame_t *frame)
 int
 resolve_gfid_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *buf, dict_t *xattr, struct iatt *postparent)
+                        struct iatt *buf, dict_t *xdata,
+                        struct iatt *postparent)
 {
         server_state_t       *state = NULL;
         server_resolve_t     *resolve = NULL;
@@ -111,7 +103,7 @@ out:
 int
 resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-                  dict_t *xattr, struct iatt *postparent)
+                  dict_t *xdata, struct iatt *postparent)
 {
         server_state_t       *state = NULL;
         server_resolve_t     *resolve = NULL;
@@ -155,7 +147,8 @@ resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     (char **) &resolve_loc->path);
 
         STACK_WIND (frame, resolve_gfid_entry_cbk,
-                    BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
+                    frame->root->client->bound_xl,
+                    frame->root->client->bound_xl->fops->lookup,
                     &resolve->resolve_loc, NULL);
         return 0;
 out:
@@ -178,25 +171,21 @@ resolve_gfid (call_frame_t *frame)
         resolve = state->resolve_now;
         resolve_loc = &resolve->resolve_loc;
 
-        if (!uuid_is_null (resolve->pargfid)) {
+        if (!uuid_is_null (resolve->pargfid))
                 uuid_copy (resolve_loc->gfid, resolve->pargfid);
-                resolve_loc->inode = inode_new (state->itable);
-                ret = inode_path (resolve_loc->inode, NULL,
-                                  (char **)&resolve_loc->path);
-        } else if (!uuid_is_null (resolve->gfid)) {
+        else if (!uuid_is_null (resolve->gfid))
                 uuid_copy (resolve_loc->gfid, resolve->gfid);
-                resolve_loc->inode = inode_new (state->itable);
-                ret = inode_path (resolve_loc->inode, NULL,
-                                  (char **)&resolve_loc->path);
-        }
+
+        resolve_loc->inode = inode_new (state->itable);
+        ret = loc_path (resolve_loc, NULL);
 
         STACK_WIND (frame, resolve_gfid_cbk,
-                    BOUND_XL (frame), BOUND_XL (frame)->fops->lookup,
+                    frame->root->client->bound_xl,
+                    frame->root->client->bound_xl->fops->lookup,
                     &resolve->resolve_loc, NULL);
         return 0;
 }
 
-
 int
 resolve_continue (call_frame_t *frame)
 {
@@ -257,7 +246,7 @@ resolve_entry_simple (call_frame_t *frame)
                 /* simple resolution is indecisive. need to perform
                    deep resolution */
                 resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
+                resolve->op_errno = ESTALE;
                 ret = 1;
                 goto out;
         }
@@ -354,7 +343,7 @@ resolve_inode_simple (call_frame_t *frame)
 
         if (!inode) {
                 resolve->op_ret   = -1;
-                resolve->op_errno = ENOENT;
+                resolve->op_errno = ESTALE;
                 ret = 1;
                 goto out;
         }
@@ -426,6 +415,10 @@ out:
         if (inode)
                 inode_unref (inode);
 
+        if (ret != 0)
+                gf_log ("server", GF_LOG_WARNING, "inode for the gfid (%s) is "
+                        "not found. anonymous fd creation failed",
+                        uuid_utoa (resolve->gfid));
         return ret;
 }
 
@@ -458,23 +451,34 @@ server_resolve_anonfd (call_frame_t *frame)
 int
 server_resolve_fd (call_frame_t *frame)
 {
-        server_state_t       *state = NULL;
-        server_resolve_t     *resolve = NULL;
-        server_connection_t  *conn = NULL;
-        uint64_t              fd_no = -1;
+        server_ctx_t         *serv_ctx = NULL;
+        server_state_t       *state    = NULL;
+        client_t             *client   = NULL;
+        server_resolve_t     *resolve  = NULL;
+        uint64_t              fd_no    = -1;
 
         state = CALL_STATE (frame);
         resolve = state->resolve_now;
-        conn  = SERVER_CONNECTION (frame);
 
         fd_no = resolve->fd_no;
 
-        if (fd_no == -2) {
+        if (fd_no == GF_ANON_FD_NO) {
                 server_resolve_anonfd (frame);
                 return 0;
         }
 
-        state->fd = gf_fd_fdptr_get (conn->fdtable, fd_no);
+        client = frame->root->client;
+
+        serv_ctx = server_ctx_get (client, client->this);
+
+        if (serv_ctx == NULL) {
+                gf_log ("", GF_LOG_INFO, "server_ctx_get() failed");
+                resolve->op_ret   = -1;
+                resolve->op_errno = ENOMEM;
+                return 0;
+        }
+
+        state->fd = gf_fd_fdptr_get (serv_ctx->fdtable, fd_no);
 
         if (!state->fd) {
                 gf_log ("", GF_LOG_INFO, "fd not found in context");
@@ -529,14 +533,12 @@ int
 server_resolve_done (call_frame_t *frame)
 {
         server_state_t    *state = NULL;
-        xlator_t          *bound_xl = NULL;
 
         state = CALL_STATE (frame);
-        bound_xl = BOUND_XL (frame);
 
         server_print_request (frame);
 
-        state->resume_fn (frame, bound_xl);
+        state->resume_fn (frame, frame->root->client->bound_xl);
 
         return 0;
 }
diff --git a/xlators/protocol/server/src/server-rpc-fops.c b/xlators/protocol/server/src/server-rpc-fops.c
new file mode 100644
index 000000000..8bdadec6b
--- /dev/null
+++ b/xlators/protocol/server/src/server-rpc-fops.c
@@ -0,0 +1,6182 @@
+/*
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <openssl/md5.h>
+
+#include "server.h"
+#include "server-helpers.h"
+#include "glusterfs3-xdr.h"
+#include "glusterfs3.h"
+#include "compat-errno.h"
+
+#include "xdr-nfs3.h"
+
+#define SERVER_REQ_SET_ERROR(req, ret)                          \
+        do {                                                    \
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);      \
+                ret = RPCSVC_ACTOR_ERROR;                       \
+        } while (0)
+
+/* Callback function section */
+int
+server_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                   dict_t *xdata)
+{
+        gfs3_statfs_rsp      rsp    = {0,};
+        rpcsvc_request_t    *req    = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "%"PRId64": STATFS (%s)",
+                        frame->root->unique, strerror (op_errno));
+                goto out;
+        }
+
+        gf_statfs_from_statfs (&rsp.statfs, buf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_statfs_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *stbuf, dict_t *xdata,
+                   struct iatt *postparent)
+{
+        rpcsvc_request_t    *req        = NULL;
+        server_state_t      *state      = NULL;
+        inode_t             *root_inode = NULL;
+        inode_t             *link_inode = NULL;
+        loc_t                fresh_loc  = {0,};
+        gfs3_lookup_rsp      rsp        = {0,};
+        uuid_t               rootgfid   = {0,};
+
+        state = CALL_STATE (frame);
+
+        if (state->is_revalidate == 1 && op_ret == -1) {
+                state->is_revalidate = 2;
+                loc_copy (&fresh_loc, &state->loc);
+                inode_unref (fresh_loc.inode);
+                fresh_loc.inode = inode_new (state->itable);
+
+                STACK_WIND (frame, server_lookup_cbk,
+                            frame->root->client->bound_xl,
+                            frame->root->client->bound_xl->fops->lookup,
+                            &fresh_loc, state->xdata);
+
+                loc_wipe (&fresh_loc);
+                return 0;
+        }
+
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                if (state->is_revalidate && op_errno == ENOENT) {
+                        if (!__is_root_gfid (state->resolve.gfid)) {
+                                inode_unlink (state->loc.inode,
+                                              state->loc.parent,
+                                              state->loc.name);
+                        }
+                }
+                goto out;
+        }
+
+        root_inode = frame->root->client->bound_xl->itable->root;
+        if (inode == root_inode) {
+                /* we just looked up root ("/") */
+                stbuf->ia_ino = 1;
+                rootgfid[15]  = 1;
+                uuid_copy (stbuf->ia_gfid, rootgfid);
+                if (inode->ia_type == 0)
+                        inode->ia_type = stbuf->ia_type;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+
+        if (!__is_root_gfid (inode->gfid)) {
+                link_inode = inode_link (inode, state->loc.parent,
+                                         state->loc.name, stbuf);
+                if (link_inode) {
+                        inode_lookup (link_inode);
+                        inode_unref (link_inode);
+                }
+        }
+
+out:
+        rsp.op_ret   = op_ret;
+        rsp.op_errno = gf_errno_to_error (op_errno);
+
+        if (op_ret) {
+                if (state->resolve.bname) {
+                        gf_log (this->name, ((op_errno == ENOENT) ?
+                                             GF_LOG_TRACE : GF_LOG_INFO),
+                                "%"PRId64": LOOKUP %s (%s/%s) ==> "
+                                "(%s)", frame->root->unique,
+                                state->loc.path,
+                                uuid_utoa (state->resolve.pargfid),
+                                state->resolve.bname,
+                                strerror (op_errno));
+                } else {
+                        gf_log (this->name, ((op_errno == ENOENT) ?
+                                             GF_LOG_TRACE : GF_LOG_INFO),
+                                "%"PRId64": LOOKUP %s (%s) ==> (%s)",
+                                frame->root->unique, state->loc.path,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+        }
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_lookup_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+               dict_t *xdata)
+{
+        gfs3_lk_rsp          rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        state = CALL_STATE (frame);
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%"PRId64": LK %"PRId64" (%s) ==> "
+                                "(%s)", frame->root->unique,
+                                state->resolve.fd_no,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+        switch (lock->l_type) {
+        case F_RDLCK:
+                lock->l_type = GF_LK_F_RDLCK;
+                break;
+        case F_WRLCK:
+                lock->l_type = GF_LK_F_WRLCK;
+                break;
+        case F_UNLCK:
+                lock->l_type = GF_LK_F_UNLCK;
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unknown lock type: %"PRId32"!", lock->l_type);
+                break;
+        }
+
+        gf_proto_flock_from_flock (&rsp.flock, lock);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_lk_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp    rsp        = {0,};
+        server_state_t   *state     = NULL;
+        rpcsvc_request_t *req       = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        gf_log (this->name, (op_errno == ENOENT)?
+                                GF_LOG_DEBUG:GF_LOG_ERROR,
+                                "%"PRId64": INODELK %s (%s) ==> (%s)",
+                                frame->root->unique, state->loc.path,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp       = {0,};
+        server_state_t   *state     = NULL;
+        rpcsvc_request_t *req       = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%"PRId64": FINODELK %"PRId64" (%s) "
+                                "==> (%s)", frame->root->unique,
+                                state->resolve.fd_no,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp       = {0,};
+        server_state_t   *state     = NULL;
+        rpcsvc_request_t *req       = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%"PRId64": ENTRYLK %s (%s) ==> (%s)",
+                                frame->root->unique, state->loc.path,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp       = {0,};
+        server_state_t   *state     = NULL;
+        rpcsvc_request_t *req       = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%"PRId64": FENTRYLK %"PRId64" (%s) ==>(%s)",
+                                frame->root->unique, state->resolve.fd_no,
+                                uuid_utoa (state->resolve.gfid),
+                                strerror (op_errno));
+                }
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req   = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": ACCESS %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_rmdir_rsp       rsp    = {0,};
+        server_state_t      *state  = NULL;
+        inode_t             *parent = NULL;
+        rpcsvc_request_t    *req    = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": RMDIR %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        inode_unlink (state->loc.inode, state->loc.parent,
+                      state->loc.name);
+        parent = inode_parent (state->loc.inode, 0, NULL);
+        if (parent)
+                /* parent should not be found for directories after
+                 * inode_unlink, since directories cannot have
+                 * hardlinks.
+                 */
+                inode_unref (parent);
+        else
+                inode_forget (state->loc.inode, 0);
+
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_rmdir_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_mkdir_rsp       rsp        = {0,};
+        server_state_t      *state      = NULL;
+        inode_t             *link_inode = NULL;
+        rpcsvc_request_t    *req        = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": MKDIR %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        link_inode = inode_link (inode, state->loc.parent,
+                                 state->loc.name, stbuf);
+        inode_lookup (link_inode);
+        inode_unref (link_inode);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_mkdir_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_mknod_rsp       rsp        = {0,};
+        server_state_t      *state      = NULL;
+        inode_t             *link_inode = NULL;
+        rpcsvc_request_t    *req        = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": MKNOD %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        link_inode = inode_link (inode, state->loc.parent,
+                                 state->loc.name, stbuf);
+        inode_lookup (link_inode);
+        inode_unref (link_inode);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_mknod_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FSYNCDIR %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+        gfs3_readdir_rsp     rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+        int                  ret   = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": READDIR %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        /* (op_ret == 0) is valid, and means EOF */
+        if (op_ret) {
+                ret = serialize_rsp_dirent (entries, &rsp);
+                if (ret == -1) {
+                        op_ret   = -1;
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_readdir_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        readdir_rsp_cleanup (&rsp);
+
+        return 0;
+}
+
+int
+server_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        server_state_t      *state    = NULL;
+        server_ctx_t        *serv_ctx = NULL;
+        rpcsvc_request_t    *req      = NULL;
+        gfs3_opendir_rsp     rsp      = {0,};
+        uint64_t             fd_no    = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": OPENDIR %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        serv_ctx = server_ctx_get (frame->root->client, this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        fd_bind (fd);
+        fd_no = gf_fd_unused_get (serv_ctx->fdtable, fd);
+        fd_ref (fd); // on behalf of the client
+
+out:
+        rsp.fd = fd_no;
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_opendir_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": REMOVEXATTR %s (%s) of key %s ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        state->name, strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req   = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FREMOVEXATTR %"PRId64" (%s) (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), state->name,
+                        strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req   = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        gfs3_getxattr_rsp    rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (((op_errno == ENOTSUP) ||
+                                      (op_errno == ENODATA) ||
+                                      (op_errno == ENOENT)) ?
+                                     GF_LOG_DEBUG : GF_LOG_INFO),
+                        "%"PRId64": GETXATTR %s (%s) (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        state->name, strerror (op_errno));
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, &rsp.dict.dict_val,
+                                    rsp.dict.dict_len, op_errno, out);
+
+out:
+        rsp.op_ret        = op_ret;
+        rsp.op_errno      = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_getxattr_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
+{
+        gfs3_fgetxattr_rsp   rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (((op_errno == ENOTSUP) ||
+                                      (op_errno == ENODATA) ||
+                                      (op_errno == ENOENT)) ?
+                                     GF_LOG_DEBUG : GF_LOG_INFO),
+                        "%"PRId64": FGETXATTR %"PRId64" (%s) (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        state->name, strerror (op_errno));
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, &rsp.dict.dict_val,
+                                    rsp.dict.dict_len, op_errno, out);
+
+out:
+
+        rsp.op_ret        = op_ret;
+        rsp.op_errno      = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fgetxattr_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+/* print every key */
+static int
+_gf_server_log_setxattr_failure (dict_t *d, char *k, data_t *v,
+                                 void *tmp)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+
+        frame = tmp;
+        state = CALL_STATE (frame);
+
+        gf_log (THIS->name, GF_LOG_INFO,
+                "%"PRId64": SETXATTR %s (%s) ==> %s",
+                frame->root->unique, state->loc.path,
+                uuid_utoa (state->resolve.gfid), k);
+        return 0;
+}
+
+int
+server_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp = {0,};
+        rpcsvc_request_t *req = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                if (op_errno != ENOTSUP)
+                        dict_foreach (state->dict,
+                                      _gf_server_log_setxattr_failure,
+                                      frame);
+
+                gf_log (THIS->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_INFO),
+                        "%s", strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+/* print every key here */
+static int
+_gf_server_log_fsetxattr_failure (dict_t *d, char *k, data_t *v,
+                                 void *tmp)
+{
+        call_frame_t        *frame = NULL;
+        server_state_t      *state = NULL;
+
+        frame = tmp;
+        state = CALL_STATE (frame);
+
+        gf_log (THIS->name, GF_LOG_INFO,
+                "%"PRId64": FSETXATTR %"PRId64" (%s) ==> %s",
+                frame->root->unique, state->resolve.fd_no,
+                uuid_utoa (state->resolve.gfid), k);
+
+        return 0;
+}
+
+int
+server_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp     rsp = {0,};
+        rpcsvc_request_t *req = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret == -1) {
+                state = CALL_STATE (frame);
+                if (op_errno != ENOTSUP) {
+                        dict_foreach (state->dict,
+                                      _gf_server_log_fsetxattr_failure,
+                                      frame);
+                }
+                gf_log (THIS->name, ((op_errno == ENOTSUP) ?
+                                     GF_LOG_DEBUG : GF_LOG_INFO),
+                        "%s", strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+        gfs3_rename_rsp      rsp        = {0,};
+        server_state_t      *state      = NULL;
+        rpcsvc_request_t    *req        = NULL;
+        inode_t             *tmp_inode  = NULL;
+        inode_t             *tmp_parent = NULL;
+        char         oldpar_str[50]     = {0,};
+        char         newpar_str[50]     = {0,};
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret == -1) {
+                uuid_utoa_r (state->resolve.gfid, oldpar_str);
+                uuid_utoa_r (state->resolve2.gfid, newpar_str);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": RENAME %s (%s/%s) -> %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        oldpar_str, state->resolve.bname, state->loc2.path,
+                        newpar_str, state->resolve2.bname, strerror (op_errno));
+                goto out;
+        }
+
+        stbuf->ia_type = state->loc.inode->ia_type;
+
+        /* TODO: log gfid of the inodes */
+        gf_log (frame->root->client->bound_xl->name, GF_LOG_TRACE,
+                "%"PRId64": RENAME_CBK  %s ==> %s",
+                frame->root->unique, state->loc.name, state->loc2.name);
+
+        /* Before renaming the inode, we have to get the inode for the
+         * destination entry (i.e. inode with state->loc2.parent as
+         * parent and state->loc2.name as name). If it exists, then
+         * unlink that inode, and send forget on that inode if the
+         * unlinked entry is the last entry. In case of fuse client
+         * the fuse kernel module itself sends the forget on the
+         * unlinked inode.
+         */
+        tmp_inode = inode_grep (state->loc.inode->table,
+                                state->loc2.parent, state->loc2.name);
+        if (tmp_inode) {
+                inode_unlink (tmp_inode, state->loc2.parent,
+                              state->loc2.name);
+                tmp_parent = inode_parent (tmp_inode, 0, NULL);
+                if (tmp_parent)
+                        inode_unref (tmp_parent);
+                else
+                        inode_forget (tmp_inode, 0);
+
+                inode_unref (tmp_inode);
+        }
+
+        inode_rename (state->itable,
+                      state->loc.parent, state->loc.name,
+                      state->loc2.parent, state->loc2.name,
+                      state->loc.inode, stbuf);
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+
+        gf_stat_from_iatt (&rsp.preoldparent, preoldparent);
+        gf_stat_from_iatt (&rsp.postoldparent, postoldparent);
+
+        gf_stat_from_iatt (&rsp.prenewparent, prenewparent);
+        gf_stat_from_iatt (&rsp.postnewparent, postnewparent);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req   = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_rename_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_unlink_rsp      rsp    = {0,};
+        server_state_t      *state  = NULL;
+        inode_t             *parent = NULL;
+        rpcsvc_request_t    *req    = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret) {
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": UNLINK %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        /* TODO: log gfid of the inodes */
+        gf_log (frame->root->client->bound_xl->name, GF_LOG_TRACE,
+                "%"PRId64": UNLINK_CBK %s",
+                frame->root->unique, state->loc.name);
+
+        inode_unlink (state->loc.inode, state->loc.parent,
+                      state->loc.name);
+
+        parent = inode_parent (state->loc.inode, 0, NULL);
+        if (parent)
+                inode_unref (parent);
+        else
+                inode_forget (state->loc.inode, 0);
+
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_unlink_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_symlink_rsp     rsp        = {0,};
+        server_state_t      *state      = NULL;
+        inode_t             *link_inode = NULL;
+        rpcsvc_request_t    *req        = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": SYMLINK %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        link_inode = inode_link (inode, state->loc.parent,
+                                 state->loc.name, stbuf);
+        inode_lookup (link_inode);
+        inode_unref (link_inode);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_symlink_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        gfs3_link_rsp        rsp         = {0,};
+        server_state_t      *state       = NULL;
+        inode_t             *link_inode  = NULL;
+        rpcsvc_request_t    *req         = NULL;
+        char              gfid_str[50]   = {0,};
+        char              newpar_str[50] = {0,};
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret) {
+                uuid_utoa_r (state->resolve.gfid, gfid_str);
+                uuid_utoa_r (state->resolve2.pargfid, newpar_str);
+
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": LINK %s (%s) -> %s/%s ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        gfid_str, newpar_str, state->resolve2.bname,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+        link_inode = inode_link (inode, state->loc2.parent,
+                                 state->loc2.name, stbuf);
+        inode_unref (link_inode);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_link_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        gfs3_truncate_rsp    rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": TRUNCATE %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.prestat, prebuf);
+        gf_stat_from_iatt (&rsp.poststat, postbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_truncate_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                  dict_t *xdata)
+{
+        gfs3_fstat_rsp       rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FSTAT %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fstat_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        gfs3_ftruncate_rsp   rsp   = {0};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FTRUNCATE %"PRId64" (%s)==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.prestat, prebuf);
+        gf_stat_from_iatt (&rsp.poststat, postbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_ftruncate_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        gf_common_rsp        rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": FLUSH %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        gfs3_fsync_rsp       rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FSYNC %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&(rsp.prestat), prebuf);
+        gf_stat_from_iatt (&(rsp.poststat), postbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fsync_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        gfs3_write_rsp       rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": WRITEV %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                         uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.prestat, prebuf);
+        gf_stat_from_iatt (&rsp.poststat, postbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_write_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iovec *vector, int32_t count,
+                  struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+        gfs3_read_rsp        rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+#ifdef GF_TESTING_IO_XDATA
+        {
+                int ret = 0;
+                if (!xdata)
+                        xdata = dict_new ();
+
+                ret = dict_set_str (xdata, "testing-the-xdata-key",
+                                       "testing-xdata-value");
+        }
+#endif
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": READV %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        rsp.size = op_ret;
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, vector, count, iobref,
+                             (xdrproc_t)xdr_gfs3_read_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      uint32_t weak_checksum, uint8_t *strong_checksum,
+                      dict_t *xdata)
+{
+        gfs3_rchecksum_rsp   rsp   = {0,};
+        rpcsvc_request_t    *req   = NULL;
+        server_state_t      *state = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": RCHECKSUM %"PRId64" (%s)==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid), strerror (op_errno));
+                goto out;
+        }
+
+        rsp.weak_checksum = weak_checksum;
+
+        rsp.strong_checksum.strong_checksum_val = (char *)strong_checksum;
+        rsp.strong_checksum.strong_checksum_len = MD5_DIGEST_LENGTH;
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_rchecksum_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        server_state_t      *state    = NULL;
+        server_ctx_t        *serv_ctx = NULL;
+        rpcsvc_request_t    *req      = NULL;
+        uint64_t             fd_no    = 0;
+        gfs3_open_rsp        rsp      = {0,};
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": OPEN %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        serv_ctx = server_ctx_get (frame->root->client, this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        fd_bind (fd);
+        fd_no = gf_fd_unused_get (serv_ctx->fdtable, fd);
+        fd_ref (fd);
+        rsp.fd = fd_no;
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_open_rsp);
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *stbuf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        server_state_t      *state      = NULL;
+        server_ctx_t        *serv_ctx   = NULL;
+        inode_t             *link_inode = NULL;
+        rpcsvc_request_t    *req        = NULL;
+        uint64_t             fd_no      = 0;
+        gfs3_create_rsp      rsp        = {0,};
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        state = CALL_STATE (frame);
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": CREATE %s (%s/%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.pargfid),
+                        state->resolve.bname, strerror (op_errno));
+                goto out;
+        }
+
+        /* TODO: log gfid too */
+        gf_log (frame->root->client->bound_xl->name, GF_LOG_TRACE,
+                "%"PRId64": CREATE %s (%s)",
+                frame->root->unique, state->loc.name,
+                uuid_utoa (stbuf->ia_gfid));
+
+        link_inode = inode_link (inode, state->loc.parent,
+                                 state->loc.name, stbuf);
+
+        if (!link_inode) {
+                op_ret = -1;
+                op_errno = ENOENT;
+                goto out;
+        }
+
+        if (link_inode != inode) {
+                /*
+                  VERY racy code (if used anywhere else)
+                  -- don't do this without understanding
+                */
+
+                inode_unref (fd->inode);
+                fd->inode = inode_ref (link_inode);
+        }
+
+        inode_lookup (link_inode);
+        inode_unref (link_inode);
+
+        serv_ctx = server_ctx_get (frame->root->client, this);
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        fd_bind (fd);
+        fd_no = gf_fd_unused_get (serv_ctx->fdtable, fd);
+        fd_ref (fd);
+
+        if ((fd_no < 0) || (fd == 0)) {
+                op_ret = fd_no;
+                op_errno = errno;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+        gf_stat_from_iatt (&rsp.preparent, preparent);
+        gf_stat_from_iatt (&rsp.postparent, postparent);
+
+out:
+        rsp.fd        = fd_no;
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_create_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *buf,
+                     struct iatt *stbuf, dict_t *xdata)
+{
+        gfs3_readlink_rsp    rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": READLINK %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.buf, stbuf);
+        rsp.path = (char *)buf;
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        if (!rsp.path)
+                rsp.path = "";
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_readlink_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                 dict_t *xdata)
+{
+        gfs3_stat_rsp        rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state  = CALL_STATE (frame);
+                gf_log (this->name, (op_errno == ENOENT)?
+                        GF_LOG_DEBUG:GF_LOG_ERROR,
+                        "%"PRId64": STAT %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.stat, stbuf);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_stat_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_setattr_rsp     rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": SETATTR %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_setattr_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_fsetattr_rsp    rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state  = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FSETATTR %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fsetattr_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+        gfs3_xattrop_rsp     rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": XATTROP %s (%s) ==> (%s)",
+                        frame->root->unique, state->loc.path,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, &rsp.dict.dict_val,
+                                    rsp.dict.dict_len, op_errno, out);
+
+out:
+        rsp.op_ret        = op_ret;
+        rsp.op_errno      = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_xattrop_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        gfs3_xattrop_rsp     rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FXATTROP %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, &rsp.dict.dict_val,
+                                    rsp.dict.dict_len, op_errno, out);
+
+out:
+        rsp.op_ret        = op_ret;
+        rsp.op_errno      = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_fxattrop_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+int
+server_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
+{
+        gfs3_readdirp_rsp    rsp   = {0,};
+        server_state_t      *state = NULL;
+        rpcsvc_request_t    *req   = NULL;
+        int                  ret   = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret < 0) {
+                state = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": READDIRP %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        /* (op_ret == 0) is valid, and means EOF */
+        if (op_ret) {
+                ret = serialize_rsp_direntp (entries, &rsp);
+                if (ret == -1) {
+                        op_ret   = -1;
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        }
+
+        /* TODO: need more clear thoughts before calling this function. */
+        /* gf_link_inodes_from_dirent (this, state->fd->inode, entries); */
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gfs3_readdirp_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        readdirp_rsp_cleanup (&rsp);
+
+        return 0;
+}
+
+int
+server_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_fallocate_rsp rsp   = {0,};
+        server_state_t    *state = NULL;
+        rpcsvc_request_t  *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state  = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": FALLOCATE %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+                            (xdrproc_t) xdr_gfs3_fallocate_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_discard_rsp rsp     = {0,};
+        server_state_t    *state = NULL;
+        rpcsvc_request_t  *req   = NULL;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, &rsp.xdata.xdata_val,
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                state  = CALL_STATE (frame);
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": DISCARD %"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        req = frame->local;
+        server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+                            (xdrproc_t) xdr_gfs3_discard_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+int
+server_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        gfs3_zerofill_rsp  rsp    = {0,};
+        server_state_t    *state  = NULL;
+        rpcsvc_request_t  *req    = NULL;
+
+        req = frame->local;
+        state  = CALL_STATE (frame);
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val),
+                                    rsp.xdata.xdata_len, op_errno, out);
+
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%"PRId64": ZEROFILL%"PRId64" (%s) ==> (%s)",
+                        frame->root->unique, state->resolve.fd_no,
+                        uuid_utoa (state->resolve.gfid),
+                        strerror (op_errno));
+                goto out;
+        }
+
+        gf_stat_from_iatt (&rsp.statpre, statpre);
+        gf_stat_from_iatt (&rsp.statpost, statpost);
+
+out:
+        rsp.op_ret    = op_ret;
+        rsp.op_errno  = gf_errno_to_error (op_errno);
+
+        server_submit_reply(frame, req, &rsp, NULL, 0, NULL,
+                            (xdrproc_t) xdr_gfs3_zerofill_rsp);
+
+        GF_FREE (rsp.xdata.xdata_val);
+
+        return 0;
+}
+
+
+/* Resume function section */
+
+int
+server_rchecksum_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state    = NULL;
+        int             op_ret   = 0;
+        int             op_errno = EINVAL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0) {
+                op_ret   = state->resolve.op_ret;
+                op_errno = state->resolve.op_errno;
+                goto err;
+        }
+
+        STACK_WIND (frame, server_rchecksum_cbk, bound_xl,
+                    bound_xl->fops->rchecksum, state->fd,
+                    state->offset, state->size, state->xdata);
+
+        return 0;
+err:
+        server_rchecksum_cbk (frame, NULL, frame->this, op_ret, op_errno, 0,
+                              NULL, NULL);
+
+        return 0;
+
+}
+
+int
+server_lk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_lk_cbk, bound_xl, bound_xl->fops->lk,
+                    state->fd, state->cmd, &state->flock, state->xdata);
+
+        return 0;
+
+err:
+        server_lk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                       state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_rename_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+        int             op_ret = 0;
+        int             op_errno = 0;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0) {
+                op_ret   = state->resolve.op_ret;
+                op_errno = state->resolve.op_errno;
+                goto err;
+        }
+
+        if (state->resolve2.op_ret != 0) {
+                op_ret   = state->resolve2.op_ret;
+                op_errno = state->resolve2.op_errno;
+                goto err;
+        }
+
+        STACK_WIND (frame, server_rename_cbk,
+                    bound_xl, bound_xl->fops->rename,
+                    &state->loc, &state->loc2, state->xdata);
+        return 0;
+err:
+        server_rename_cbk (frame, NULL, frame->this, op_ret, op_errno,
+                           NULL, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_link_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+        int             op_ret = 0;
+        int             op_errno = 0;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0) {
+                op_ret   = state->resolve.op_ret;
+                op_errno = state->resolve.op_errno;
+                goto err;
+        }
+
+        if (state->resolve2.op_ret != 0) {
+                op_ret   = state->resolve2.op_ret;
+                op_errno = state->resolve2.op_errno;
+                goto err;
+        }
+
+        state->loc2.inode = inode_ref (state->loc.inode);
+
+        STACK_WIND (frame, server_link_cbk, bound_xl, bound_xl->fops->link,
+                    &state->loc, &state->loc2, state->xdata);
+
+        return 0;
+err:
+        server_link_cbk (frame, NULL, frame->this, op_ret, op_errno,
+                         NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+server_symlink_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->loc.inode = inode_new (state->itable);
+
+        STACK_WIND (frame, server_symlink_cbk,
+                    bound_xl, bound_xl->fops->symlink,
+                    state->name, &state->loc, state->umask, state->xdata);
+
+        return 0;
+err:
+        server_symlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_access_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_access_cbk,
+                    bound_xl, bound_xl->fops->access,
+                    &state->loc, state->mask, state->xdata);
+        return 0;
+err:
+        server_access_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL);
+        return 0;
+}
+
+int
+server_fentrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        GF_UNUSED  int  ret   = -1;
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (state->xdata)
+                ret = dict_set_str (state->xdata, "connection-id",
+                                    frame->root->client->client_uid);
+
+        STACK_WIND (frame, server_fentrylk_cbk, bound_xl,
+                    bound_xl->fops->fentrylk,
+                    state->volume, state->fd, state->name,
+                    state->cmd, state->type, state->xdata);
+
+        return 0;
+err:
+        server_fentrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL);
+        return 0;
+}
+
+
+int
+server_entrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        GF_UNUSED int   ret   = -1;
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (state->xdata)
+                ret = dict_set_str (state->xdata, "connection-id",
+                                    frame->root->client->client_uid);
+
+        STACK_WIND (frame, server_entrylk_cbk,
+                    bound_xl, bound_xl->fops->entrylk,
+                    state->volume, &state->loc, state->name,
+                    state->cmd, state->type, state->xdata);
+        return 0;
+err:
+        server_entrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL);
+        return 0;
+}
+
+
+int
+server_finodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        GF_UNUSED int   ret   = -1;
+        server_state_t *state = NULL;
+
+        gf_log (bound_xl->name, GF_LOG_DEBUG, "frame %p, xlator %p",
+                frame, bound_xl);
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (state->xdata)
+                ret = dict_set_str (state->xdata, "connection-id",
+                                    frame->root->client->client_uid);
+
+        STACK_WIND (frame, server_finodelk_cbk, bound_xl,
+                    bound_xl->fops->finodelk, state->volume, state->fd,
+                    state->cmd, &state->flock, state->xdata);
+
+        return 0;
+err:
+        server_finodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL);
+
+        return 0;
+}
+
+int
+server_inodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        GF_UNUSED int   ret   = -1;
+        server_state_t *state = NULL;
+
+        gf_log (bound_xl->name, GF_LOG_DEBUG, "frame %p, xlator %p",
+                frame, bound_xl);
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (state->xdata)
+                ret = dict_set_str (state->xdata, "connection-id",
+                                    frame->root->client->client_uid);
+
+        STACK_WIND (frame, server_inodelk_cbk, bound_xl,
+                    bound_xl->fops->inodelk, state->volume, &state->loc,
+                    state->cmd, &state->flock, state->xdata);
+        return 0;
+err:
+        server_inodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL);
+        return 0;
+}
+
+int
+server_rmdir_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_rmdir_cbk, bound_xl, bound_xl->fops->rmdir,
+                    &state->loc, state->flags, state->xdata);
+        return 0;
+err:
+        server_rmdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+server_mkdir_resume (call_frame_t *frame, xlator_t *bound_xl)
+
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->loc.inode = inode_new (state->itable);
+
+        STACK_WIND (frame, server_mkdir_cbk,
+                    bound_xl, bound_xl->fops->mkdir,
+                    &(state->loc), state->mode, state->umask, state->xdata);
+
+        return 0;
+err:
+        server_mkdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_mknod_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->loc.inode = inode_new (state->itable);
+
+        STACK_WIND (frame, server_mknod_cbk,
+                    bound_xl, bound_xl->fops->mknod,
+                    &(state->loc), state->mode, state->dev,
+                    state->umask, state->xdata);
+
+        return 0;
+err:
+        server_mknod_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_fsyncdir_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fsyncdir_cbk,
+                    bound_xl,
+                    bound_xl->fops->fsyncdir,
+                    state->fd, state->flags, state->xdata);
+        return 0;
+
+err:
+        server_fsyncdir_cbk (frame, NULL, frame->this,
+                             state->resolve.op_ret,
+                             state->resolve.op_errno, NULL);
+        return 0;
+}
+
+
+int
+server_readdir_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t  *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        GF_ASSERT (state->fd);
+
+        STACK_WIND (frame, server_readdir_cbk,
+                    bound_xl,
+                    bound_xl->fops->readdir,
+                    state->fd, state->size, state->offset, state->xdata);
+
+        return 0;
+err:
+        server_readdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_readdirp_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t  *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_readdirp_cbk, bound_xl,
+                    bound_xl->fops->readdirp, state->fd, state->size,
+                    state->offset, state->dict);
+
+        return 0;
+err:
+        server_readdirp_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_opendir_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t  *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->fd = fd_create (state->loc.inode, frame->root->pid);
+        if (!state->fd) {
+                gf_log ("server", GF_LOG_ERROR, "could not create the fd");
+                goto err;
+        }
+
+        STACK_WIND (frame, server_opendir_cbk,
+                    bound_xl, bound_xl->fops->opendir,
+                    &state->loc, state->fd, state->xdata);
+        return 0;
+err:
+        server_opendir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_statfs_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t      *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret !=0)
+                goto err;
+
+        STACK_WIND (frame, server_statfs_cbk,
+                    bound_xl, bound_xl->fops->statfs,
+                    &state->loc, state->xdata);
+        return 0;
+
+err:
+        server_statfs_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_removexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_removexattr_cbk,
+                    bound_xl, bound_xl->fops->removexattr,
+                    &state->loc, state->name, state->xdata);
+        return 0;
+err:
+        server_removexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                                state->resolve.op_errno, NULL);
+        return 0;
+}
+
+int
+server_fremovexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fremovexattr_cbk,
+                    bound_xl, bound_xl->fops->fremovexattr,
+                    state->fd, state->name, state->xdata);
+        return 0;
+err:
+        server_fremovexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                                 state->resolve.op_errno, NULL);
+        return 0;
+}
+
+int
+server_fgetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fgetxattr_cbk,
+                    bound_xl, bound_xl->fops->fgetxattr,
+                    state->fd, state->name, state->xdata);
+        return 0;
+err:
+        server_fgetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                              state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_xattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_xattrop_cbk,
+                    bound_xl, bound_xl->fops->xattrop,
+                    &state->loc, state->flags, state->dict, state->xdata);
+        return 0;
+err:
+        server_xattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_fxattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fxattrop_cbk,
+                    bound_xl, bound_xl->fops->fxattrop,
+                    state->fd, state->flags, state->dict, state->xdata);
+        return 0;
+err:
+        server_fxattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_fsetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_setxattr_cbk,
+                    bound_xl, bound_xl->fops->fsetxattr,
+                    state->fd, state->dict, state->flags, state->xdata);
+        return 0;
+err:
+        server_fsetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                              state->resolve.op_errno, NULL);
+
+        return 0;
+}
+
+int
+server_unlink_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_unlink_cbk,
+                    bound_xl, bound_xl->fops->unlink,
+                    &state->loc, state->flags, state->xdata);
+        return 0;
+err:
+        server_unlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+server_truncate_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_truncate_cbk,
+                    bound_xl, bound_xl->fops->truncate,
+                    &state->loc, state->offset, state->xdata);
+        return 0;
+err:
+        server_truncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+
+int
+server_fstat_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t     *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fstat_cbk,
+                    bound_xl, bound_xl->fops->fstat,
+                    state->fd, state->xdata);
+        return 0;
+err:
+        server_fstat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_setxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_setxattr_cbk,
+                    bound_xl, bound_xl->fops->setxattr,
+                    &state->loc, state->dict, state->flags, state->xdata);
+        return 0;
+err:
+        server_setxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+server_getxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_getxattr_cbk,
+                    bound_xl, bound_xl->fops->getxattr,
+                    &state->loc, state->name, state->xdata);
+        return 0;
+err:
+        server_getxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_ftruncate_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_ftruncate_cbk,
+                    bound_xl, bound_xl->fops->ftruncate,
+                    state->fd, state->offset, state->xdata);
+        return 0;
+err:
+        server_ftruncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                              state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+server_flush_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_flush_cbk,
+                    bound_xl, bound_xl->fops->flush, state->fd, state->xdata);
+        return 0;
+err:
+        server_flush_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+server_fsync_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fsync_cbk,
+                    bound_xl, bound_xl->fops->fsync,
+                    state->fd, state->flags, state->xdata);
+        return 0;
+err:
+        server_fsync_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t   *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_writev_cbk,
+                    bound_xl, bound_xl->fops->writev,
+                    state->fd, state->payload_vector, state->payload_count,
+                    state->offset, state->flags, state->iobref, state->xdata);
+
+        return 0;
+err:
+        server_writev_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_readv_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_readv_cbk,
+                    bound_xl, bound_xl->fops->readv,
+                    state->fd, state->size, state->offset, state->flags, state->xdata);
+
+        return 0;
+err:
+        server_readv_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                          state->resolve.op_errno, NULL, 0, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_create_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->loc.inode = inode_new (state->itable);
+
+        state->fd = fd_create (state->loc.inode, frame->root->pid);
+        if (!state->fd) {
+                gf_log ("server", GF_LOG_ERROR, "fd creation for the inode %s "
+                        "failed", state->loc.inode?
+                        uuid_utoa (state->loc.inode->gfid):NULL);
+                state->resolve.op_ret = -1;
+                state->resolve.op_errno = ENOMEM;
+                goto err;
+        }
+        state->fd->flags = state->flags;
+
+        STACK_WIND (frame, server_create_cbk,
+                    bound_xl, bound_xl->fops->create,
+                    &(state->loc), state->flags, state->mode,
+                    state->umask, state->fd, state->xdata);
+
+        return 0;
+err:
+        server_create_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL,
+                           NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_open_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t  *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        state->fd = fd_create (state->loc.inode, frame->root->pid);
+        state->fd->flags = state->flags;
+
+        STACK_WIND (frame, server_open_cbk,
+                    bound_xl, bound_xl->fops->open,
+                    &state->loc, state->flags, state->fd, state->xdata);
+
+        return 0;
+err:
+        server_open_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                         state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_readlink_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_readlink_cbk,
+                    bound_xl, bound_xl->fops->readlink,
+                    &state->loc, state->size, state->xdata);
+        return 0;
+err:
+        server_readlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int
+server_fsetattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fsetattr_cbk,
+                    bound_xl, bound_xl->fops->fsetattr,
+                    state->fd, &state->stbuf, state->valid, state->xdata);
+        return 0;
+err:
+        server_fsetattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+server_setattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_setattr_cbk,
+                    bound_xl, bound_xl->fops->setattr,
+                    &state->loc, &state->stbuf, state->valid, state->xdata);
+        return 0;
+err:
+        server_setattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                            state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+server_stat_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_stat_cbk,
+                    bound_xl, bound_xl->fops->stat, &state->loc, state->xdata);
+        return 0;
+err:
+        server_stat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                         state->resolve.op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+server_lookup_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t    *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        if (!state->loc.inode)
+                state->loc.inode = inode_new (state->itable);
+        else
+                state->is_revalidate = 1;
+
+        STACK_WIND (frame, server_lookup_cbk,
+                    bound_xl, bound_xl->fops->lookup,
+                    &state->loc, state->xdata);
+
+        return 0;
+err:
+        server_lookup_cbk (frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+server_fallocate_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_fallocate_cbk,
+                    bound_xl, bound_xl->fops->fallocate,
+                    state->fd, state->flags, state->offset, state->size,
+                    state->xdata);
+        return 0;
+err:
+        server_fallocate_cbk(frame, NULL, frame->this, state->resolve.op_ret,
+                             state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+server_discard_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_discard_cbk,
+                    bound_xl, bound_xl->fops->discard,
+                    state->fd, state->offset, state->size, state->xdata);
+        return 0;
+err:
+        server_discard_cbk(frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+server_zerofill_resume (call_frame_t *frame, xlator_t *bound_xl)
+{
+        server_state_t *state = NULL;
+
+        state = CALL_STATE (frame);
+
+        if (state->resolve.op_ret != 0)
+                goto err;
+
+        STACK_WIND (frame, server_zerofill_cbk,
+                    bound_xl, bound_xl->fops->zerofill,
+                    state->fd, state->offset, state->size, state->xdata);
+        return 0;
+err:
+        server_zerofill_cbk(frame, NULL, frame->this, state->resolve.op_ret,
+                           state->resolve.op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
+/* Fop section */
+
+int
+server3_3_stat (rpcsvc_request_t *req)
+{
+        server_state_t *state    = NULL;
+        call_frame_t   *frame    = NULL;
+        gfs3_stat_req   args     = {{0,},};
+        int             ret      = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return 0;
+
+        /* Initialize args first, then decode */
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_stat_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_STAT;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+
+        ret = 0;
+        resolve_and_resume (frame, server_stat_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_setattr (rpcsvc_request_t *req)
+{
+        server_state_t   *state                 = NULL;
+        call_frame_t     *frame                 = NULL;
+        gfs3_setattr_req  args                  = {{0,},};
+        int               ret                   = -1;
+        int               op_errno = 0;
+
+        if (!req)
+                return 0;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_setattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_SETATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        gf_stat_to_iatt (&args.stbuf, &state->stbuf);
+        state->valid = args.valid;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_setattr_resume);
+
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        free (args.xdata.xdata_val);
+
+        return ret;
+}
+
+
+int
+server3_3_fsetattr (rpcsvc_request_t *req)
+{
+        server_state_t    *state = NULL;
+        call_frame_t      *frame = NULL;
+        gfs3_fsetattr_req  args  = {0,};
+        int                ret   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fsetattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSETATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+
+        gf_stat_to_iatt (&args.stbuf, &state->stbuf);
+        state->valid = args.valid;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fsetattr_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_fallocate(rpcsvc_request_t *req)
+{
+        server_state_t    *state = NULL;
+        call_frame_t      *frame = NULL;
+        gfs3_fallocate_req args  = {{0},};
+        int                ret   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fallocate_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FALLOCATE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+
+        state->flags = args.flags;
+        state->offset = args.offset;
+        state->size = args.size;
+        memcpy(state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fallocate_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_discard(rpcsvc_request_t *req)
+{
+        server_state_t    *state = NULL;
+        call_frame_t      *frame = NULL;
+        gfs3_discard_req args    = {{0},};
+        int                ret   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_discard_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_DISCARD;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+
+        state->offset = args.offset;
+        state->size = args.size;
+        memcpy(state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_discard_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_zerofill(rpcsvc_request_t *req)
+{
+        server_state_t       *state      = NULL;
+        call_frame_t         *frame      = NULL;
+        gfs3_zerofill_req     args       = {{0},};
+        int                   ret        = -1;
+        int                   op_errno   = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_zerofill_req);
+        if (ret < 0) {
+                /*failed to decode msg*/;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                /* something wrong, mostly insufficient memory*/
+                req->rpc_err = GARBAGE_ARGS; /* TODO */
+                goto out;
+        }
+        frame->root->op = GF_FOP_ZEROFILL;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+
+        state->offset = args.offset;
+        state->size = args.size;
+        memcpy(state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl, state->xdata,
+                                      (args.xdata.xdata_val),
+                                      (args.xdata.xdata_len), ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_zerofill_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                req->rpc_err = GARBAGE_ARGS;
+
+        return ret;
+}
+
+int
+server3_3_readlink (rpcsvc_request_t *req)
+{
+        server_state_t    *state                 = NULL;
+        call_frame_t      *frame                 = NULL;
+        gfs3_readlink_req  args                  = {{0,},};
+        int                ret                   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_readlink_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_READLINK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        state->size  = args.size;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_readlink_resume);
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_create (rpcsvc_request_t *req)
+{
+        server_state_t  *state    = NULL;
+        call_frame_t    *frame    = NULL;
+        gfs3_create_req  args     = {{0,},};
+        int              ret      = -1;
+        int              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_create_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_CREATE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+
+        state->resolve.bname  = gf_strdup (args.bname);
+        state->mode           = args.mode;
+        state->umask          = args.umask;
+        state->flags          = gf_flags_to_flags (args.flags);
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+
+        if (state->flags & O_EXCL) {
+                state->resolve.type = RESOLVE_NOT;
+        } else {
+                state->resolve.type = RESOLVE_DONTCARE;
+        }
+
+        /* TODO: can do alloca for xdata field instead of stdalloc */
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_create_resume);
+
+out:
+        /* memory allocated by libc, don't use GF_FREE */
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_open (rpcsvc_request_t *req)
+{
+        server_state_t *state                 = NULL;
+        call_frame_t   *frame                 = NULL;
+        gfs3_open_req   args                  = {{0,},};
+        int             ret                   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_open_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_OPEN;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        state->flags = gf_flags_to_flags (args.flags);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_open_resume);
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        free (args.xdata.xdata_val);
+
+        return ret;
+}
+
+
+int
+server3_3_readv (rpcsvc_request_t *req)
+{
+        server_state_t *state = NULL;
+        call_frame_t   *frame = NULL;
+        gfs3_read_req   args  = {{0,},};
+        int             ret   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                goto out;
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_read_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_READ;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+        state->size           = args.size;
+        state->offset         = args.offset;
+        state->flags          = args.flag;
+
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_readv_resume);
+out:
+        /* memory allocated by libc, don't use GF_FREE */
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_writev (rpcsvc_request_t *req)
+{
+        server_state_t      *state  = NULL;
+        call_frame_t        *frame  = NULL;
+        gfs3_write_req       args   = {{0,},};
+        ssize_t              len    = 0;
+        int                  i      = 0;
+        int                  ret    = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        len = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_write_req);
+        if (len < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_WRITE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->offset        = args.offset;
+        state->size          = args.size;
+        state->flags         = args.flag;
+        state->iobref        = iobref_ref (req->iobref);
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (len < req->msg[0].iov_len) {
+                state->payload_vector[0].iov_base
+                        = (req->msg[0].iov_base + len);
+                state->payload_vector[0].iov_len
+                        = req->msg[0].iov_len - len;
+                state->payload_count = 1;
+        }
+
+        for (i = 1; i < req->count; i++) {
+                state->payload_vector[state->payload_count++]
+                        = req->msg[i];
+        }
+
+        for (i = 0; i < state->payload_count; i++) {
+                state->size += state->payload_vector[i].iov_len;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+#ifdef GF_TESTING_IO_XDATA
+        dict_dump (state->xdata);
+#endif
+
+        ret = 0;
+        resolve_and_resume (frame, server_writev_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+#define SERVER3_3_VECWRITE_START 0
+#define SERVER3_3_VECWRITE_READING_HDR 1
+#define SERVER3_3_VECWRITE_READING_OPAQUE 2
+
+int
+server3_3_writev_vecsizer (int state, ssize_t *readsize, char *base_addr,
+                        char *curr_addr)
+{
+        ssize_t         size = 0;
+        int             nextstate = 0;
+        gfs3_write_req  write_req = {{0,},};
+        XDR             xdr;
+
+        switch (state) {
+        case SERVER3_3_VECWRITE_START:
+                size = xdr_sizeof ((xdrproc_t) xdr_gfs3_write_req,
+                                   &write_req);
+                *readsize = size;
+                nextstate = SERVER3_3_VECWRITE_READING_HDR;
+                break;
+        case SERVER3_3_VECWRITE_READING_HDR:
+                size = xdr_sizeof ((xdrproc_t) xdr_gfs3_write_req,
+                                           &write_req);
+
+                xdrmem_create (&xdr, base_addr, size, XDR_DECODE);
+
+                /* This will fail if there is xdata sent from client, if not,
+                   well and good */
+                xdr_gfs3_write_req (&xdr, &write_req);
+
+                /* need to round off to proper roof (%4), as XDR packing pads
+                   the end of opaque object with '0' */
+                size = roof (write_req.xdata.xdata_len, 4);
+
+                *readsize = size;
+
+                if (!size)
+                        nextstate = SERVER3_3_VECWRITE_START;
+                else
+                        nextstate = SERVER3_3_VECWRITE_READING_OPAQUE;
+
+                free (write_req.xdata.xdata_val);
+
+                break;
+
+        case SERVER3_3_VECWRITE_READING_OPAQUE:
+                *readsize = 0;
+                nextstate = SERVER3_3_VECWRITE_START;
+                break;
+        default:
+                gf_log ("server", GF_LOG_ERROR, "wrong state: %d", state);
+        }
+
+        return nextstate;
+}
+
+
+int
+server3_3_release (rpcsvc_request_t *req)
+{
+        client_t         *client   = NULL;
+        server_ctx_t     *serv_ctx = NULL;
+        gfs3_release_req  args     = {{0,},};
+        gf_common_rsp     rsp      = {0,};
+        int               ret      = -1;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_release_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        client = req->trans->xl_private;
+        if (!client) {
+                /* Handshake is not complete yet. */
+                req->rpc_err = SYSTEM_ERR;
+                goto out;
+        }
+
+        serv_ctx = server_ctx_get (client, client->this);
+        if (serv_ctx == NULL) {
+                gf_log (req->trans->name, GF_LOG_INFO,
+                        "server_ctx_get() failed");
+                req->rpc_err = SYSTEM_ERR;
+                goto out;
+        }
+
+        gf_fd_put (serv_ctx->fdtable, args.fd);
+
+        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+server3_3_releasedir (rpcsvc_request_t *req)
+{
+        client_t            *client   = NULL;
+        server_ctx_t        *serv_ctx = NULL;
+        gfs3_releasedir_req  args     = {{0,},};
+        gf_common_rsp        rsp      = {0,};
+        int                  ret      = -1;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_release_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        client = req->trans->xl_private;
+        if (!client) {
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        serv_ctx = server_ctx_get (client, client->this);
+        if (serv_ctx == NULL) {
+                gf_log (req->trans->name, GF_LOG_INFO,
+                        "server_ctx_get() failed");
+                req->rpc_err = SYSTEM_ERR;
+                goto out;
+        }
+
+        gf_fd_put (serv_ctx->fdtable, args.fd);
+
+        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+server3_3_fsync (rpcsvc_request_t *req)
+{
+        server_state_t *state = NULL;
+        call_frame_t   *frame = NULL;
+        gfs3_fsync_req  args  = {{0,},};
+        int             ret   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fsync_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSYNC;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->flags         = args.data;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fsync_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_flush (rpcsvc_request_t *req)
+{
+        server_state_t *state = NULL;
+        call_frame_t   *frame = NULL;
+        gfs3_flush_req  args  = {{0,},};
+        int             ret   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_flush_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FLUSH;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_flush_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_ftruncate (rpcsvc_request_t *req)
+{
+        server_state_t     *state = NULL;
+        call_frame_t       *frame = NULL;
+        gfs3_ftruncate_req  args  = {{0,},};
+        int                 ret   = -1;
+        int                 op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_ftruncate_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FTRUNCATE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+        state->offset         = args.offset;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_ftruncate_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_fstat (rpcsvc_request_t *req)
+{
+        server_state_t *state = NULL;
+        call_frame_t   *frame = NULL;
+        gfs3_fstat_req  args  = {{0,},};
+        int             ret   = -1;
+        int             op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fstat_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSTAT;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        state->resolve.fd_no   = args.fd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fstat_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_truncate (rpcsvc_request_t *req)
+{
+        server_state_t    *state                 = NULL;
+        call_frame_t      *frame                 = NULL;
+        gfs3_truncate_req  args                  = {{0,},};
+        int                ret                   = -1;
+        int                op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_truncate_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_TRUNCATE;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+        state->offset        = args.offset;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_truncate_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_unlink (rpcsvc_request_t *req)
+{
+        server_state_t  *state                  = NULL;
+        call_frame_t    *frame                  = NULL;
+        gfs3_unlink_req  args                   = {{0,},};
+        int              ret                    = -1;
+        int              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_unlink_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_UNLINK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.bname  = gf_strdup (args.bname);
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+
+        state->flags = args.xflags;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_unlink_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_setxattr (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        dict_t              *dict                  = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_setxattr_req    args                  = {{0,},};
+        int32_t              ret                   = -1;
+        int32_t              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.dict.dict_val = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_setxattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_SETXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type     = RESOLVE_MUST;
+        state->flags            = args.flags;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+        state->dict = dict;
+
+        /* There can be some commands hidden in key, check and proceed */
+        gf_server_check_setxattr_cmd (frame, dict);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_setxattr_resume);
+
+        return ret;
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+
+int
+server3_3_fsetxattr (rpcsvc_request_t *req)
+{
+        server_state_t      *state                = NULL;
+        dict_t              *dict                 = NULL;
+        call_frame_t        *frame                = NULL;
+        gfs3_fsetxattr_req   args                 = {{0,},};
+        int32_t              ret                  = -1;
+        int32_t              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.dict.dict_val = alloca (req->msg[0].iov_len);
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fsetxattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSETXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type      = RESOLVE_MUST;
+        state->resolve.fd_no     = args.fd;
+        state->flags             = args.flags;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+        state->dict = dict;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fsetxattr_resume);
+
+        return ret;
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+
+int
+server3_3_fxattrop (rpcsvc_request_t *req)
+{
+        dict_t              *dict                 = NULL;
+        server_state_t      *state                = NULL;
+        call_frame_t        *frame                = NULL;
+        gfs3_fxattrop_req    args                 = {{0,},};
+        int32_t              ret                  = -1;
+        int32_t              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.dict.dict_val = alloca (req->msg[0].iov_len);
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fxattrop_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FXATTROP;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        state->resolve.fd_no   = args.fd;
+        state->flags           = args.flags;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+        state->dict = dict;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fxattrop_resume);
+
+        return ret;
+
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+
+int
+server3_3_xattrop (rpcsvc_request_t *req)
+{
+        dict_t              *dict                  = NULL;
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_xattrop_req     args                  = {{0,},};
+        int32_t              ret                   = -1;
+        int32_t              op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.dict.dict_val = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_xattrop_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_XATTROP;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        state->flags           = args.flags;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+        state->dict = dict;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_xattrop_resume);
+
+        return ret;
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+int
+server3_3_getxattr (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_getxattr_req    args                  = {{0,},};
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_getxattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_GETXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (args.namelen) {
+                state->name = gf_strdup (args.name);
+                /* There can be some commands hidden in key, check and proceed */
+                gf_server_check_getxattr_cmd (frame, state->name);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_getxattr_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_fgetxattr (rpcsvc_request_t *req)
+{
+        server_state_t      *state      = NULL;
+        call_frame_t        *frame      = NULL;
+        gfs3_fgetxattr_req   args       = {{0,},};
+        int                  ret        = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name = alloca (256);
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fgetxattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FGETXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (args.namelen)
+                state->name = gf_strdup (args.name);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fgetxattr_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_removexattr (rpcsvc_request_t *req)
+{
+        server_state_t       *state                 = NULL;
+        call_frame_t         *frame                 = NULL;
+        gfs3_removexattr_req  args                  = {{0,},};
+        int                   ret                   = -1;
+        int                   op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_removexattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_REMOVEXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+        state->name           = gf_strdup (args.name);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_removexattr_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_fremovexattr (rpcsvc_request_t *req)
+{
+        server_state_t       *state                 = NULL;
+        call_frame_t         *frame                 = NULL;
+        gfs3_fremovexattr_req  args                  = {{0,},};
+        int                   ret                   = -1;
+        int                   op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name = alloca (4096);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fremovexattr_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FREMOVEXATTR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.fd_no  = args.fd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+        state->name           = gf_strdup (args.name);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fremovexattr_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+
+int
+server3_3_opendir (rpcsvc_request_t *req)
+{
+        server_state_t   *state                 = NULL;
+        call_frame_t     *frame                 = NULL;
+        gfs3_opendir_req  args                  = {{0,},};
+        int               ret                   = -1;
+        int               op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_opendir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_OPENDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_opendir_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_readdirp (rpcsvc_request_t *req)
+{
+        server_state_t      *state        = NULL;
+        call_frame_t        *frame        = NULL;
+        gfs3_readdirp_req    args         = {{0,},};
+        size_t               headers_size = 0;
+        int                  ret          = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_readdirp_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_READDIRP;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        /* FIXME: this should go away when variable sized iobufs are introduced
+         * and transport layer can send msgs bigger than current page-size.
+         */
+        headers_size = sizeof (struct rpc_msg) + sizeof (gfs3_readdir_rsp);
+        if ((frame->this->ctx->page_size < args.size)
+            || ((frame->this->ctx->page_size - args.size) < headers_size)) {
+                state->size = frame->this->ctx->page_size - headers_size;
+        } else {
+                state->size   = args.size;
+        }
+
+        state->resolve.type = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->offset = args.offset;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        /* here, dict itself works as xdata */
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->dict,
+                                      (args.dict.dict_val),
+                                      (args.dict.dict_len), ret,
+                                      op_errno, out);
+
+
+        ret = 0;
+        resolve_and_resume (frame, server_readdirp_resume);
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        free (args.dict.dict_val);
+
+        return ret;
+}
+
+int
+server3_3_readdir (rpcsvc_request_t *req)
+{
+        server_state_t      *state        = NULL;
+        call_frame_t        *frame        = NULL;
+        gfs3_readdir_req     args         = {{0,},};
+        size_t               headers_size = 0;
+        int                  ret          = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_readdir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_READDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        /* FIXME: this should go away when variable sized iobufs are introduced
+         * and transport layer can send msgs bigger than current page-size.
+         */
+        headers_size = sizeof (struct rpc_msg) + sizeof (gfs3_readdir_rsp);
+        if ((frame->this->ctx->page_size < args.size)
+            || ((frame->this->ctx->page_size - args.size) < headers_size)) {
+                state->size = frame->this->ctx->page_size - headers_size;
+        } else {
+                state->size   = args.size;
+        }
+
+        state->resolve.type = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->offset = args.offset;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_readdir_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_fsyncdir (rpcsvc_request_t *req)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+        gfs3_fsyncdir_req    args  = {{0,},};
+        int                  ret   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fsyncdir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FSYNCDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type = RESOLVE_MUST;
+        state->resolve.fd_no = args.fd;
+        state->flags = args.data;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fsyncdir_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_mknod (rpcsvc_request_t *req)
+{
+        server_state_t      *state                  = NULL;
+        call_frame_t        *frame                  = NULL;
+        gfs3_mknod_req       args                   = {{0,},};
+        int                  ret                    = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_mknod_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_MKNOD;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_NOT;
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+        state->resolve.bname   = gf_strdup (args.bname);
+
+        state->mode  = args.mode;
+        state->dev   = args.dev;
+        state->umask = args.umask;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_mknod_resume);
+
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        /* memory allocated by libc, don't use GF_FREE */
+        free (args.xdata.xdata_val);
+
+        return ret;
+
+}
+
+
+int
+server3_3_mkdir (rpcsvc_request_t *req)
+{
+        server_state_t      *state                  = NULL;
+        call_frame_t        *frame                  = NULL;
+        gfs3_mkdir_req       args                   = {{0,},};
+        int                  ret                    = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_mkdir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_MKDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_NOT;
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+        state->resolve.bname   = gf_strdup (args.bname);
+
+        state->mode  = args.mode;
+        state->umask = args.umask;
+
+        /* TODO: can do alloca for xdata field instead of stdalloc */
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_mkdir_resume);
+
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        free (args.xdata.xdata_val);
+
+        return ret;
+}
+
+
+int
+server3_3_rmdir (rpcsvc_request_t *req)
+{
+        server_state_t      *state                  = NULL;
+        call_frame_t        *frame                  = NULL;
+        gfs3_rmdir_req       args                   = {{0,},};
+        int                  ret                    = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_rmdir_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_RMDIR;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+        state->resolve.bname   = gf_strdup (args.bname);
+
+        state->flags = args.xflags;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_rmdir_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_inodelk (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_inodelk_req     args                  = {{0,},};
+        int                  cmd                   = 0;
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.volume = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_inodelk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_INODELK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_EXACT;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        cmd = args.cmd;
+        switch (cmd) {
+        case GF_LK_GETLK:
+                state->cmd = F_GETLK;
+                break;
+        case GF_LK_SETLK:
+                state->cmd = F_SETLK;
+                break;
+        case GF_LK_SETLKW:
+                state->cmd = F_SETLKW;
+                break;
+        }
+
+        state->type = args.type;
+        state->volume = gf_strdup (args.volume);
+
+        gf_proto_flock_to_flock (&args.flock, &state->flock);
+
+        switch (state->type) {
+        case GF_LK_F_RDLCK:
+                state->flock.l_type = F_RDLCK;
+                break;
+        case GF_LK_F_WRLCK:
+                state->flock.l_type = F_WRLCK;
+                break;
+        case GF_LK_F_UNLCK:
+                state->flock.l_type = F_UNLCK;
+                break;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_inodelk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        free (args.flock.lk_owner.lk_owner_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_finodelk (rpcsvc_request_t *req)
+{
+        server_state_t      *state        = NULL;
+        call_frame_t        *frame        = NULL;
+        gfs3_finodelk_req    args         = {{0,},};
+        int                  ret          = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.volume = alloca (256);
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_finodelk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FINODELK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type = RESOLVE_EXACT;
+        state->volume = gf_strdup (args.volume);
+        state->resolve.fd_no = args.fd;
+        state->cmd = args.cmd;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        switch (state->cmd) {
+        case GF_LK_GETLK:
+                state->cmd = F_GETLK;
+                break;
+        case GF_LK_SETLK:
+                state->cmd = F_SETLK;
+                break;
+        case GF_LK_SETLKW:
+                state->cmd = F_SETLKW;
+                break;
+        }
+
+        state->type = args.type;
+
+        gf_proto_flock_to_flock (&args.flock, &state->flock);
+
+        switch (state->type) {
+        case GF_LK_F_RDLCK:
+                state->flock.l_type = F_RDLCK;
+                break;
+        case GF_LK_F_WRLCK:
+                state->flock.l_type = F_WRLCK;
+                break;
+        case GF_LK_F_UNLCK:
+                state->flock.l_type = F_UNLCK;
+                break;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_finodelk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        free (args.flock.lk_owner.lk_owner_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_entrylk (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_entrylk_req     args                  = {{0,},};
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.volume = alloca (256);
+        args.name   = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_entrylk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_ENTRYLK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_EXACT;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (args.namelen)
+                state->name   = gf_strdup (args.name);
+        state->volume         = gf_strdup (args.volume);
+
+        state->cmd            = args.cmd;
+        state->type           = args.type;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_entrylk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_fentrylk (rpcsvc_request_t *req)
+{
+        server_state_t      *state        = NULL;
+        call_frame_t        *frame        = NULL;
+        gfs3_fentrylk_req    args         = {{0,},};
+        int                  ret          = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.name   = alloca (256);
+        args.volume = alloca (256);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_fentrylk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_FENTRYLK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type = RESOLVE_EXACT;
+        state->resolve.fd_no = args.fd;
+        state->cmd  = args.cmd;
+        state->type = args.type;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        if (args.namelen)
+                state->name = gf_strdup (args.name);
+        state->volume = gf_strdup (args.volume);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_fentrylk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_access (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_access_req      args                  = {{0,},};
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_access_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_ACCESS;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+        state->mask          = args.mask;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_access_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+
+int
+server3_3_symlink (rpcsvc_request_t *req)
+{
+        server_state_t      *state                 = NULL;
+        call_frame_t        *frame                 = NULL;
+        gfs3_symlink_req     args                  = {{0,},};
+        int                  ret                   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.bname    = alloca (req->msg[0].iov_len);
+        args.linkname = alloca (4096);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_symlink_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_SYMLINK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_NOT;
+        memcpy (state->resolve.pargfid, args.pargfid, 16);
+        state->resolve.bname  = gf_strdup (args.bname);
+        state->name           = gf_strdup (args.linkname);
+        state->umask          = args.umask;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_symlink_resume);
+
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        /* memory allocated by libc, don't use GF_FREE */
+        free (args.xdata.xdata_val);
+
+        return ret;
+}
+
+
+
+int
+server3_3_link (rpcsvc_request_t *req)
+{
+        server_state_t      *state                     = NULL;
+        call_frame_t        *frame                     = NULL;
+        gfs3_link_req        args                      = {{0,},};
+        int                  ret                       = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.newbname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_link_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_LINK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type    = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.oldgfid, 16);
+
+        state->resolve2.type   = RESOLVE_NOT;
+        state->resolve2.bname  = gf_strdup (args.newbname);
+        memcpy (state->resolve2.pargfid, args.newgfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_link_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_rename (rpcsvc_request_t *req)
+{
+        server_state_t      *state                     = NULL;
+        call_frame_t        *frame                     = NULL;
+        gfs3_rename_req      args                      = {{0,},};
+        int                  ret                       = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        args.oldbname = alloca (req->msg[0].iov_len);
+        args.newbname = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_rename_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_RENAME;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        state->resolve.bname  = gf_strdup (args.oldbname);
+        memcpy (state->resolve.pargfid, args.oldgfid, 16);
+
+        state->resolve2.type  = RESOLVE_MAY;
+        state->resolve2.bname = gf_strdup (args.newbname);
+        memcpy (state->resolve2.pargfid, args.newgfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_rename_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server3_3_lk (rpcsvc_request_t *req)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+        gfs3_lk_req          args  = {{0,},};
+        int                  ret   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_lk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_LK;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.fd_no = args.fd;
+        state->cmd =  args.cmd;
+        state->type = args.type;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        switch (state->cmd) {
+        case GF_LK_GETLK:
+                state->cmd = F_GETLK;
+                break;
+        case GF_LK_SETLK:
+                state->cmd = F_SETLK;
+                break;
+        case GF_LK_SETLKW:
+                state->cmd = F_SETLKW;
+                break;
+        case GF_LK_RESLK_LCK:
+                state->cmd = F_RESLK_LCK;
+                break;
+        case GF_LK_RESLK_LCKW:
+                state->cmd = F_RESLK_LCKW;
+                break;
+        case GF_LK_RESLK_UNLCK:
+                state->cmd = F_RESLK_UNLCK;
+                break;
+        case GF_LK_GETLK_FD:
+                state->cmd = F_GETLK_FD;
+                break;
+
+        }
+
+
+        gf_proto_flock_to_flock (&args.flock, &state->flock);
+
+        switch (state->type) {
+        case GF_LK_F_RDLCK:
+                state->flock.l_type = F_RDLCK;
+                break;
+        case GF_LK_F_WRLCK:
+                state->flock.l_type = F_WRLCK;
+                break;
+        case GF_LK_F_UNLCK:
+                state->flock.l_type = F_UNLCK;
+                break;
+        default:
+                gf_log (frame->root->client->bound_xl->name, GF_LOG_ERROR,
+                        "fd - %"PRId64" (%s): Unknown lock type: %"PRId32"!",
+                        state->resolve.fd_no,
+                        uuid_utoa (state->fd->inode->gfid), state->type);
+                break;
+        }
+
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_lk_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        free (args.flock.lk_owner.lk_owner_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+int
+server3_3_rchecksum (rpcsvc_request_t *req)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+        gfs3_rchecksum_req   args  = {0,};
+        int                  ret   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_rchecksum_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_RCHECKSUM;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type  = RESOLVE_MAY;
+        state->resolve.fd_no = args.fd;
+        state->offset        = args.offset;
+        state->size          = args.len;
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_rchecksum_resume);
+out:
+        free (args.xdata.xdata_val);
+
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+int
+server_null (rpcsvc_request_t *req)
+{
+        gf_common_rsp rsp = {0,};
+
+        /* Accepted */
+        rsp.op_ret = 0;
+
+        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
+                             (xdrproc_t)xdr_gf_common_rsp);
+
+        return 0;
+}
+
+int
+server3_3_lookup (rpcsvc_request_t *req)
+{
+        call_frame_t        *frame    = NULL;
+        server_state_t      *state    = NULL;
+        gfs3_lookup_req      args     = {{0,},};
+        int                  ret      = -1;
+        int                  op_errno = 0;
+
+        GF_VALIDATE_OR_GOTO ("server", req, err);
+
+        args.bname           = alloca (req->msg[0].iov_len);
+        args.xdata.xdata_val = alloca (req->msg[0].iov_len);
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_lookup_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto err;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto err;
+        }
+        frame->root->op = GF_FOP_LOOKUP;
+
+        /* NOTE: lookup() uses req->ino only to identify if a lookup()
+         *       is requested for 'root' or not
+         */
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_DONTCARE;
+
+        if (args.bname && strcmp (args.bname, "")) {
+                memcpy (state->resolve.pargfid, args.pargfid, 16);
+                state->resolve.bname = gf_strdup (args.bname);
+        } else {
+                memcpy (state->resolve.gfid, args.gfid, 16);
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_lookup_resume);
+
+        return ret;
+out:
+
+        server_lookup_cbk (frame, NULL, frame->this, -1, EINVAL, NULL, NULL,
+                           NULL, NULL);
+        ret = 0;
+err:
+        return ret;
+}
+
+int
+server3_3_statfs (rpcsvc_request_t *req)
+{
+        server_state_t      *state = NULL;
+        call_frame_t        *frame = NULL;
+        gfs3_statfs_req      args  = {{0,},};
+        int                  ret   = -1;
+        int                  op_errno = 0;
+
+        if (!req)
+                return ret;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gfs3_statfs_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        frame = get_frame_from_request (req);
+        if (!frame) {
+                // something wrong, mostly insufficient memory
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+        frame->root->op = GF_FOP_STATFS;
+
+        state = CALL_STATE (frame);
+        if (!frame->root->client->bound_xl) {
+                /* auth failure, request on subvolume without setvolume */
+                SERVER_REQ_SET_ERROR (req, ret);
+                goto out;
+        }
+
+        state->resolve.type   = RESOLVE_MUST;
+        memcpy (state->resolve.gfid, args.gfid, 16);
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (frame->root->client->bound_xl,
+                                      state->xdata,
+                                      args.xdata.xdata_val,
+                                      args.xdata.xdata_len, ret,
+                                      op_errno, out);
+
+        ret = 0;
+        resolve_and_resume (frame, server_statfs_resume);
+out:
+        if (op_errno)
+                SERVER_REQ_SET_ERROR (req, ret);
+
+        return ret;
+}
+
+
+rpcsvc_actor_t glusterfs3_3_fop_actors[] = {
+        [GFS3_OP_NULL]         = {"NULL",         GFS3_OP_NULL,         server_null,            NULL, 0, DRC_NA},
+        [GFS3_OP_STAT]         = {"STAT",         GFS3_OP_STAT,         server3_3_stat,         NULL, 0, DRC_NA},
+        [GFS3_OP_READLINK]     = {"READLINK",     GFS3_OP_READLINK,     server3_3_readlink,     NULL, 0, DRC_NA},
+        [GFS3_OP_MKNOD]        = {"MKNOD",        GFS3_OP_MKNOD,        server3_3_mknod,        NULL, 0, DRC_NA},
+        [GFS3_OP_MKDIR]        = {"MKDIR",        GFS3_OP_MKDIR,        server3_3_mkdir,        NULL, 0, DRC_NA},
+        [GFS3_OP_UNLINK]       = {"UNLINK",       GFS3_OP_UNLINK,       server3_3_unlink,       NULL, 0, DRC_NA},
+        [GFS3_OP_RMDIR]        = {"RMDIR",        GFS3_OP_RMDIR,        server3_3_rmdir,        NULL, 0, DRC_NA},
+        [GFS3_OP_SYMLINK]      = {"SYMLINK",      GFS3_OP_SYMLINK,      server3_3_symlink,      NULL, 0, DRC_NA},
+        [GFS3_OP_RENAME]       = {"RENAME",       GFS3_OP_RENAME,       server3_3_rename,       NULL, 0, DRC_NA},
+        [GFS3_OP_LINK]         = {"LINK",         GFS3_OP_LINK,         server3_3_link,         NULL, 0, DRC_NA},
+        [GFS3_OP_TRUNCATE]     = {"TRUNCATE",     GFS3_OP_TRUNCATE,     server3_3_truncate,     NULL, 0, DRC_NA},
+        [GFS3_OP_OPEN]         = {"OPEN",         GFS3_OP_OPEN,         server3_3_open,         NULL, 0, DRC_NA},
+        [GFS3_OP_READ]         = {"READ",         GFS3_OP_READ,         server3_3_readv,        NULL, 0, DRC_NA},
+        [GFS3_OP_WRITE]        = {"WRITE",        GFS3_OP_WRITE,        server3_3_writev,       server3_3_writev_vecsizer, 0, DRC_NA},
+        [GFS3_OP_STATFS]       = {"STATFS",       GFS3_OP_STATFS,       server3_3_statfs,       NULL, 0, DRC_NA},
+        [GFS3_OP_FLUSH]        = {"FLUSH",        GFS3_OP_FLUSH,        server3_3_flush,        NULL, 0, DRC_NA},
+        [GFS3_OP_FSYNC]        = {"FSYNC",        GFS3_OP_FSYNC,        server3_3_fsync,        NULL, 0, DRC_NA},
+        [GFS3_OP_SETXATTR]     = {"SETXATTR",     GFS3_OP_SETXATTR,     server3_3_setxattr,     NULL, 0, DRC_NA},
+        [GFS3_OP_GETXATTR]     = {"GETXATTR",     GFS3_OP_GETXATTR,     server3_3_getxattr,     NULL, 0, DRC_NA},
+        [GFS3_OP_REMOVEXATTR]  = {"REMOVEXATTR",  GFS3_OP_REMOVEXATTR,  server3_3_removexattr,  NULL, 0, DRC_NA},
+        [GFS3_OP_OPENDIR]      = {"OPENDIR",      GFS3_OP_OPENDIR,      server3_3_opendir,      NULL, 0, DRC_NA},
+        [GFS3_OP_FSYNCDIR]     = {"FSYNCDIR",     GFS3_OP_FSYNCDIR,     server3_3_fsyncdir,     NULL, 0, DRC_NA},
+        [GFS3_OP_ACCESS]       = {"ACCESS",       GFS3_OP_ACCESS,       server3_3_access,       NULL, 0, DRC_NA},
+        [GFS3_OP_CREATE]       = {"CREATE",       GFS3_OP_CREATE,       server3_3_create,       NULL, 0, DRC_NA},
+        [GFS3_OP_FTRUNCATE]    = {"FTRUNCATE",    GFS3_OP_FTRUNCATE,    server3_3_ftruncate,    NULL, 0, DRC_NA},
+        [GFS3_OP_FSTAT]        = {"FSTAT",        GFS3_OP_FSTAT,        server3_3_fstat,        NULL, 0, DRC_NA},
+        [GFS3_OP_LK]           = {"LK",           GFS3_OP_LK,           server3_3_lk,           NULL, 0, DRC_NA},
+        [GFS3_OP_LOOKUP]       = {"LOOKUP",       GFS3_OP_LOOKUP,       server3_3_lookup,       NULL, 0, DRC_NA},
+        [GFS3_OP_READDIR]      = {"READDIR",      GFS3_OP_READDIR,      server3_3_readdir,      NULL, 0, DRC_NA},
+        [GFS3_OP_INODELK]      = {"INODELK",      GFS3_OP_INODELK,      server3_3_inodelk,      NULL, 0, DRC_NA},
+        [GFS3_OP_FINODELK]     = {"FINODELK",     GFS3_OP_FINODELK,     server3_3_finodelk,     NULL, 0, DRC_NA},
+        [GFS3_OP_ENTRYLK]      = {"ENTRYLK",      GFS3_OP_ENTRYLK,      server3_3_entrylk,      NULL, 0, DRC_NA},
+        [GFS3_OP_FENTRYLK]     = {"FENTRYLK",     GFS3_OP_FENTRYLK,     server3_3_fentrylk,     NULL, 0, DRC_NA},
+        [GFS3_OP_XATTROP]      = {"XATTROP",      GFS3_OP_XATTROP,      server3_3_xattrop,      NULL, 0, DRC_NA},
+        [GFS3_OP_FXATTROP]     = {"FXATTROP",     GFS3_OP_FXATTROP,     server3_3_fxattrop,     NULL, 0, DRC_NA},
+        [GFS3_OP_FGETXATTR]    = {"FGETXATTR",    GFS3_OP_FGETXATTR,    server3_3_fgetxattr,    NULL, 0, DRC_NA},
+        [GFS3_OP_FSETXATTR]    = {"FSETXATTR",    GFS3_OP_FSETXATTR,    server3_3_fsetxattr,    NULL, 0, DRC_NA},
+        [GFS3_OP_RCHECKSUM]    = {"RCHECKSUM",    GFS3_OP_RCHECKSUM,    server3_3_rchecksum,    NULL, 0, DRC_NA},
+        [GFS3_OP_SETATTR]      = {"SETATTR",      GFS3_OP_SETATTR,      server3_3_setattr,      NULL, 0, DRC_NA},
+        [GFS3_OP_FSETATTR]     = {"FSETATTR",     GFS3_OP_FSETATTR,     server3_3_fsetattr,     NULL, 0, DRC_NA},
+        [GFS3_OP_READDIRP]     = {"READDIRP",     GFS3_OP_READDIRP,     server3_3_readdirp,     NULL, 0, DRC_NA},
+        [GFS3_OP_RELEASE]      = {"RELEASE",      GFS3_OP_RELEASE,      server3_3_release,      NULL, 0, DRC_NA},
+        [GFS3_OP_RELEASEDIR]   = {"RELEASEDIR",   GFS3_OP_RELEASEDIR,   server3_3_releasedir,   NULL, 0, DRC_NA},
+        [GFS3_OP_FREMOVEXATTR] = {"FREMOVEXATTR", GFS3_OP_FREMOVEXATTR, server3_3_fremovexattr, NULL, 0, DRC_NA},
+        [GFS3_OP_FALLOCATE]    = {"FALLOCATE",    GFS3_OP_FALLOCATE,    server3_3_fallocate,    NULL, 0, DRC_NA},
+        [GFS3_OP_DISCARD]      = {"DISCARD",      GFS3_OP_DISCARD,      server3_3_discard,      NULL, 0, DRC_NA},
+        [GFS3_OP_ZEROFILL]    =  {"ZEROFILL",     GFS3_OP_ZEROFILL,     server3_3_zerofill,     NULL, 0, DRC_NA},
+};
+
+
+struct rpcsvc_program glusterfs3_3_fop_prog = {
+        .progname  = "GlusterFS 3.3",
+        .prognum   = GLUSTER_FOP_PROGRAM,
+        .progver   = GLUSTER_FOP_VERSION,
+        .numactors = GLUSTER_FOP_PROCCNT,
+        .actors    = glusterfs3_3_fop_actors,
+};
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 7ac8590a4..378aa393a 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -34,27 +25,56 @@
 #include "statedump.h"
 #include "defaults.h"
 #include "authenticate.h"
-#include "rpcsvc.h"
 
 void
 grace_time_handler (void *data)
 {
-        server_connection_t     *conn = NULL;
-        xlator_t                *this = NULL;
-        gf_boolean_t            cancelled = _gf_false;
+        client_t      *client    = NULL;
+        xlator_t      *this      = NULL;
+        gf_timer_t    *timer     = NULL;
+        server_ctx_t  *serv_ctx  = NULL;
+        gf_boolean_t   cancelled = _gf_false;
+        gf_boolean_t   detached  = _gf_false;
 
-        conn = data;
-        this = conn->this;
+        client = data;
+        this = client->this;
 
-        GF_VALIDATE_OR_GOTO (THIS->name, conn, out);
         GF_VALIDATE_OR_GOTO (THIS->name, this, out);
 
-        gf_log (this->name, GF_LOG_INFO, "grace timer expired");
+        gf_log (this->name, GF_LOG_INFO, "grace timer expired for %s",
+                client->client_uid);
 
-        cancelled = server_cancel_conn_timer (this, conn);
+        serv_ctx = server_ctx_get (client, this);
+
+        if (serv_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
+                goto out;
+        }
+
+        LOCK (&serv_ctx->fdtable_lock);
+        {
+                if (serv_ctx->grace_timer) {
+                        timer = serv_ctx->grace_timer;
+                        serv_ctx->grace_timer = NULL;
+                }
+        }
+        UNLOCK (&serv_ctx->fdtable_lock);
+        if (timer) {
+                gf_timer_call_cancel (this->ctx, timer);
+                cancelled = _gf_true;
+        }
         if (cancelled) {
-                server_connection_cleanup (this, conn);
-                server_conn_unref (conn);
+
+                /*
+                 * client must not be destroyed in gf_client_put(),
+                 * so take a ref.
+                 */
+                gf_client_ref (client);
+                gf_client_put (client, &detached);
+                if (detached)//reconnection did not happen :-(
+                        server_connection_cleanup (this, client,
+                                                   INTERNAL_LOCKS | POSIX_LOCKS);
+                gf_client_unref (client);
         }
 out:
         return;
@@ -122,16 +142,20 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
         struct iovec            rsp        = {0,};
         server_state_t         *state      = NULL;
         char                    new_iobref = 0;
-        server_connection_t    *conn  = NULL;
+        client_t               *client     = NULL;
+        gf_boolean_t            lk_heal    = _gf_false;
 
         GF_VALIDATE_OR_GOTO ("server", req, ret);
 
         if (frame) {
                 state = CALL_STATE (frame);
                 frame->local = NULL;
-                conn  = SERVER_CONNECTION(frame);
+                client = frame->root->client;
         }
 
+        if (client)
+                lk_heal = ((server_conf_t *) client->this->private)->lk_heal;
+
         if (!iobref) {
                 iobref = iobref_new ();
                 if (!iobref) {
@@ -164,8 +188,15 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
         iobuf_unref (iob);
         if (ret == -1) {
                 gf_log_callingfn ("", GF_LOG_ERROR, "Reply submission failed");
-                if (frame && conn)
-                        server_connection_cleanup (frame->this, conn);
+                if (frame && client && !lk_heal) {
+                        server_connection_cleanup (frame->this, client,
+                                                  INTERNAL_LOCKS | POSIX_LOCKS);
+                } else {
+                        gf_log_callingfn ("", GF_LOG_ERROR,
+                                          "Reply submission failed");
+                        /* TODO: Failure of open(dir), create, inodelk, entrylk
+                           or lk fops send failure must be handled specially. */
+                }
                 goto ret;
         }
 
@@ -176,8 +207,7 @@ ret:
         }
 
         if (frame) {
-                if (frame->root->trans)
-                        server_conn_unref (frame->root->trans);
+                gf_client_unref (client);
                 STACK_DESTROY (frame->root);
         }
 
@@ -188,92 +218,6 @@ ret:
         return ret;
 }
 
-/* */
-int
-server_fd_to_dict (xlator_t *this, dict_t *dict)
-{
-        server_conf_t           *conf = NULL;
-        server_connection_t     *trav = NULL;
-        char                    key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        int                     count = 0;
-        int                     ret = -1;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        conf = this->private;
-        if (!conf)
-                return -1;
-
-        ret = pthread_mutex_trylock (&conf->mutex);
-        if (ret)
-                return -1;
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "conn%d", count++);
-                fdtable_dump_to_dict (trav->fdtable, key, dict);
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = dict_set_int32 (dict, "conncount", count);
-out:
-        return ret;
-}
-
-int
-server_fd (xlator_t *this)
-{
-        server_conf_t        *conf = NULL;
-        server_connection_t  *trav = NULL;
-        char                 key[GF_DUMP_MAX_BUF_LEN];
-        int                  i = 1;
-        int                  ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-
-        conf = this->private;
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-        }
-
-        gf_proc_dump_add_section("xlator.protocol.server.conn");
-
-        ret = pthread_mutex_trylock (&conf->mutex);
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump fdtable"
-                       " errno: %d", errno);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                if (trav->id) {
-                        gf_proc_dump_build_key(key,
-                                               "conn","%d.id", i);
-                        gf_proc_dump_write(key, "%s", trav->id);
-                }
-
-                gf_proc_dump_build_key(key,"conn","%d.ref",i)
-                        gf_proc_dump_write(key, "%d", trav->ref);
-                if (trav->bound_xl) {
-                        gf_proc_dump_build_key(key,
-                                               "conn","%d.bound_xl", i);
-                        gf_proc_dump_write(key, "%s", trav->bound_xl->name);
-                }
-
-                gf_proc_dump_build_key(key,
-                                       "conn","%d.id", i);
-                fdtable_dump(trav->fdtable,key);
-                i++;
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = 0;
-out:
-        return ret;
-}
 
 int
 server_priv_to_dict (xlator_t *this, dict_t *dict)
@@ -293,28 +237,40 @@ server_priv_to_dict (xlator_t *this, dict_t *dict)
                 return 0;
         //TODO: Dump only specific info to dict
 
-        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                peerinfo = &xprt->peerinfo;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.hostname", count);
-                ret = dict_set_str (dict, key, peerinfo->identifier);
-                if (ret)
-                        goto out;
+        pthread_mutex_lock (&conf->mutex);
+        {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        peerinfo = &xprt->peerinfo;
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "client%d.hostname",
+                                  count);
+                        ret = dict_set_str (dict, key, peerinfo->identifier);
+                        if (ret)
+                                goto unlock;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.bytesread", count);
-                ret = dict_set_uint64 (dict, key, xprt->total_bytes_read);
-                if (ret)
-                        goto out;
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "client%d.bytesread",
+                                  count);
+                        ret = dict_set_uint64 (dict, key,
+                                               xprt->total_bytes_read);
+                        if (ret)
+                                goto unlock;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.byteswrite", count);
-                ret = dict_set_uint64 (dict, key, xprt->total_bytes_write);
-                if (ret)
-                        goto out;
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "client%d.byteswrite",
+                                  count);
+                        ret = dict_set_uint64 (dict, key,
+                                               xprt->total_bytes_write);
+                        if (ret)
+                                goto unlock;
 
-                count++;
+                        count++;
+                }
         }
+unlock:
+        pthread_mutex_unlock (&conf->mutex);
+        if (ret)
+                goto out;
 
         ret = dict_set_int32 (dict, "clientcount", count);
 
@@ -338,10 +294,19 @@ server_priv (xlator_t *this)
         if (!conf)
                 return 0;
 
-        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                total_read  += xprt->total_bytes_read;
-                total_write += xprt->total_bytes_write;
+        gf_proc_dump_build_key (key, "xlator.protocol.server", "priv");
+        gf_proc_dump_add_section (key);
+
+        ret = pthread_mutex_trylock (&conf->mutex);
+        if (ret != 0)
+                goto out;
+        {
+                list_for_each_entry (xprt, &conf->xprt_list, list) {
+                        total_read  += xprt->total_bytes_read;
+                        total_write += xprt->total_bytes_write;
+                }
         }
+        pthread_mutex_unlock (&conf->mutex);
 
         gf_proc_dump_build_key(key, "server", "total-bytes-read");
         gf_proc_dump_write(key, "%"PRIu64, total_read);
@@ -351,103 +316,16 @@ server_priv (xlator_t *this)
 
         ret = 0;
 out:
-        return ret;
-}
-
-int
-server_inode_to_dict (xlator_t *this, dict_t *dict)
-{
-        server_conf_t           *conf = NULL;
-        server_connection_t     *trav = NULL;
-        char                    key[32] = {0,};
-        int                     count = 0;
-        int                     ret = -1;
-        xlator_t                *prev_bound_xl = NULL;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        conf = this->private;
-        if (!conf)
-                return -1;
-
-        ret = pthread_mutex_trylock (&conf->mutex);
         if (ret)
-                return -1;
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                if (trav->bound_xl && trav->bound_xl->itable) {
-                        /* Presently every brick contains only one
-                         * bound_xl for all connections. This will lead
-                         * to duplicating of the inode lists, if listing
-                         * is done for every connection. This simple check
-                         * prevents duplication in the present case. If
-                         * need arises the check can be improved.
-                         */
-                        if (trav->bound_xl == prev_bound_xl)
-                                continue;
-                        prev_bound_xl = trav->bound_xl;
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "conn%d", count);
-                        inode_table_dump_to_dict (trav->bound_xl->itable,
-                                                  key, dict);
-                        count++;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = dict_set_int32 (dict, "conncount", count);
+                gf_proc_dump_write ("Unable to print priv",
+                                    "(Lock acquisition failed) %s",
+                                    this?this->name:"server");
 
-out:
-        if (prev_bound_xl)
-                prev_bound_xl = NULL;
         return ret;
 }
 
-int
-server_inode (xlator_t *this)
-{
-        server_conf_t        *conf = NULL;
-        server_connection_t  *trav = NULL;
-        char                 key[GF_DUMP_MAX_BUF_LEN];
-        int                  i = 1;
-        int                  ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-
-        conf = this->private;
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "conf null in xlator");
-                return -1;
-        }
-
-        ret = pthread_mutex_trylock (&conf->mutex);
-        if (ret) {
-                gf_log("", GF_LOG_WARNING, "Unable to dump itable"
-                       " errno: %d", errno);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &conf->conns, list) {
-                if (trav->bound_xl && trav->bound_xl->itable) {
-                        gf_proc_dump_build_key(key,
-                                               "conn","%d.bound_xl.%s",
-                                               i, trav->bound_xl->name);
-                        inode_table_dump(trav->bound_xl->itable,key);
-                        i++;
-                }
-        }
-        pthread_mutex_unlock (&conf->mutex);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-static void
+static int
 get_auth_types (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t   *auth_dict = NULL;
@@ -484,82 +362,88 @@ get_auth_types (dict_t *this, char *key, data_t *value, void *data)
 
         GF_FREE (key_cpy);
 out:
-        return;
+        return 0;
 }
 
+int
+_check_for_auth_option (dict_t *d, char *k, data_t *v,
+                        void *tmp)
+{
+        int       ret           = 0;
+        xlator_t *xl            = NULL;
+        char     *tail          = NULL;
+        char     *tmp_addr_list = NULL;
+        char     *addr          = NULL;
+        char     *tmp_str       = NULL;
+
+        xl = tmp;
+
+        tail = strtail (k, "auth.");
+        if (!tail)
+                goto out;
+
+        /* fast fwd thru module type */
+        tail = strchr (tail, '.');
+        if (!tail)
+                goto out;
+        tail++;
+
+        tail = strtail (tail, xl->name);
+        if (!tail)
+                goto out;
+
+        if (*tail == '.') {
+                /* when we are here, the key is checked for
+                 * valid auth.allow.<xlator>
+                 * Now we verify the ip address
+                 */
+                if (!strcmp (v->data, "*")) {
+                        ret = 0;
+                        goto out;
+                }
+
+                tmp_addr_list = gf_strdup (v->data);
+                addr = strtok_r (tmp_addr_list, ",", &tmp_str);
+                if (!addr)
+                        addr = v->data;
+
+                while (addr) {
+                        if (valid_internet_address (addr, _gf_true)) {
+                                ret = 0;
+                        } else {
+                                ret = -1;
+                                gf_log (xl->name, GF_LOG_ERROR,
+                                        "internet address '%s'"
+                                        " does not conform to"
+                                        " standards.", addr);
+                                goto out;
+                        }
+                        if (tmp_str)
+                                addr = strtok_r (NULL, ",", &tmp_str);
+                        else
+                                addr = NULL;
+                }
+
+                GF_FREE (tmp_addr_list);
+                tmp_addr_list = NULL;
+        }
+out:
+        return ret;
+}
 
 int
 validate_auth_options (xlator_t *this, dict_t *dict)
 {
         int            error = -1;
         xlator_list_t *trav = NULL;
-        data_pair_t   *pair = NULL;
-        char          *tail = NULL;
-        char          *addr = NULL;
-        char          *tmp_str = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", this, out);
         GF_VALIDATE_OR_GOTO ("server", dict, out);
 
         trav = this->children;
         while (trav) {
-                error = -1;
-                for (pair = dict->members_list; pair; pair = pair->next) {
-                        tail = strtail (pair->key, "auth.");
-                        if (!tail)
-                                continue;
-                        /* fast fwd thru module type */
-                        tail = strchr (tail, '.');
-                        if (!tail)
-                                continue;
-                        tail++;
-
-                        tail = strtail (tail, trav->xlator->name);
-                        if (!tail)
-                                continue;
-
-                        if (*tail == '.') {
-                                error = 0;
-
-                                /* when we are here, the key is checked for
-                                 * valid auth.allow.<xlator>
-                                 * Now we verify the ip address
-                                 */
-                                if (!strcmp (pair->value->data, "*")) {
-                                     error = 0;
-                                     goto out;
-                                }
-
-                                addr = strtok_r (pair->value->data, ",",
-                                                &tmp_str);
-                                if (!addr)
-                                        addr = pair->value->data;
-
-                                while (addr) {
-
-                                        if (valid_internet_address (addr) ||
-                                        valid_wildcard_internet_address (addr))
-                                        {
-                                                error = 0;
-                                        } else {
-                                                error = -1;
-                                                gf_log (this->name, GF_LOG_ERROR,
-                                                        "internet address '%s'"
-                                                        " does not conform to"
-                                                        " standards.", addr);
-                                                goto out;
-
-                                        }
-                                        if (tmp_str)
-                                                addr = strtok_r (NULL, ",",
-                                                                 &tmp_str);
-                                        else
-                                                addr = NULL;
-                                }
-
-                        }
-
-                }
+                error = dict_foreach (dict, _check_for_auth_option,
+                                      trav->xlator);
 
                 if (-1 == error) {
                         gf_log (this->name, GF_LOG_ERROR,
@@ -580,10 +464,12 @@ int
 server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
                    void *data)
 {
+        gf_boolean_t         detached   = _gf_false;
         xlator_t            *this       = NULL;
-        rpc_transport_t     *xprt       = NULL;
-        server_connection_t *conn       = NULL;
+        rpc_transport_t     *trans      = NULL;
         server_conf_t       *conf       = NULL;
+        client_t            *client     = NULL;
+        server_ctx_t        *serv_ctx   = NULL;
 
         if (!xl || !data) {
                 gf_log_callingfn ("server", GF_LOG_WARNING,
@@ -592,7 +478,7 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
         }
 
         this = xl;
-        xprt = data;
+        trans = data;
         conf = this->private;
 
         switch (event) {
@@ -600,47 +486,81 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
         {
                 /* Have a structure per new connection */
                 /* TODO: Should we create anything here at all ? * /
-                   conn = create_server_conn_state (this, xprt);
-                   if (!conn)
+                   client->conn = create_server_conn_state (this, trans);
+                   if (!client->conn)
                    goto out;
 
-                   xprt->protocol_private = conn;
+                   trans->protocol_private = client->conn;
                 */
-                INIT_LIST_HEAD (&xprt->list);
+                INIT_LIST_HEAD (&trans->list);
 
-                list_add_tail (&xprt->list, &conf->xprt_list);
+                pthread_mutex_lock (&conf->mutex);
+                {
+                        list_add_tail (&trans->list, &conf->xprt_list);
+                }
+                pthread_mutex_unlock (&conf->mutex);
 
                 break;
         }
         case RPCSVC_EVENT_DISCONNECT:
-                conn = get_server_conn_state (this, xprt);
-                if (!conn)
+                /* transport has to be removed from the list upon disconnect
+                 * irrespective of whether lock self heal is off or on, since
+                 * new transport will be created upon reconnect.
+                 */
+                pthread_mutex_lock (&conf->mutex);
+                {
+                        list_del_init (&trans->list);
+                }
+                pthread_mutex_unlock (&conf->mutex);
+
+                client = trans->xl_private;
+                if (!client)
                         break;
 
-                put_server_conn_state (this, xprt);
                 gf_log (this->name, GF_LOG_INFO, "disconnecting connection"
-                        "from %s", xprt->peerinfo.identifier);
-                list_del (&xprt->list);
+                        "from %s", client->client_uid);
+
+                /* If lock self heal is off, then destroy the
+                   conn object, else register a grace timer event */
+                if (!conf->lk_heal) {
+                        gf_client_ref (client);
+                        gf_client_put (client, &detached);
+                        if (detached)
+                                server_connection_cleanup (this, client,
+                                                           INTERNAL_LOCKS | POSIX_LOCKS);
+                        gf_client_unref (client);
+                        break;
+                }
+                trans->xl_private = NULL;
+                server_connection_cleanup (this, client, INTERNAL_LOCKS);
 
-                pthread_mutex_lock (&conn->lock);
+                serv_ctx = server_ctx_get (client, this);
+
+                if (serv_ctx == NULL) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "server_ctx_get() failed");
+                        goto out;
+                }
+
+                LOCK (&serv_ctx->fdtable_lock);
                 {
-                        if (conn->timer)
-                                goto unlock;
+                        if (!serv_ctx->grace_timer) {
 
-                        gf_log (this->name, GF_LOG_INFO, "starting a grace "
-                                "timer for %s", xprt->name);
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "starting a grace timer for %s",
+                                        client->client_uid);
 
-                        conn->timer = gf_timer_call_after (this->ctx,
-                                                           conf->grace_tv,
-                                                           grace_time_handler,
-                                                           conn);
+                                serv_ctx->grace_timer =
+                                        gf_timer_call_after (this->ctx,
+                                                             conf->grace_ts,
+                                                             grace_time_handler,
+                                                             client);
+                        }
                 }
-        unlock:
-                pthread_mutex_unlock (&conn->lock);
-
+                UNLOCK (&serv_ctx->fdtable_lock);
                 break;
         case RPCSVC_EVENT_TRANSPORT_DESTROY:
-                /*- conn obj has been disassociated from xprt on first
+                /*- conn obj has been disassociated from trans on first
                  *  disconnect.
                  *  conn cleanup and destruction is handed over to
                  *  grace_time_handler or the subsequent handler that 'owns'
@@ -673,39 +593,41 @@ out:
 }
 
 
-static void
-_delete_auth_opt (dict_t *this,
-                char *key,
-                data_t *value,
-                void *data)
+static int
+_delete_auth_opt (dict_t *this, char *key, data_t *value, void *data)
 {
         char *auth_option_pattern[] = { "auth.addr.*.allow",
-                                        "auth.addr.*.reject"};
-        if (fnmatch ( auth_option_pattern[0], key, 0) != 0) {
-                dict_del (this, key);
-                return;
-        }
+                                        "auth.addr.*.reject",
+                                        NULL};
+        int i = 0;
 
-        if (fnmatch ( auth_option_pattern[1], key, 0) != 0) {
-                dict_del (this, key);
-                return;
+        for (i = 0; auth_option_pattern[i]; i++) {
+                if (fnmatch (auth_option_pattern[i], key, 0) == 0) {
+                        dict_del (this, key);
+                        break;
+                }
         }
+
+        return 0;
 }
 
 
-static void
-_copy_auth_opt (dict_t *unused,
-                char *key,
-                data_t *value,
-                void *xl_dict)
+static int
+_copy_auth_opt (dict_t *unused, char *key, data_t *value, void *xl_dict)
 {
         char *auth_option_pattern[] = { "auth.addr.*.allow",
-                                        "auth.addr.*.reject"};
-        if (fnmatch ( auth_option_pattern[0], key, 0) != 0)
-                dict_set ((dict_t *)xl_dict, key, (value));
+                                        "auth.addr.*.reject",
+                                        NULL};
+        int i = 0;
+
+        for (i = 0; auth_option_pattern [i]; i++) {
+                if (fnmatch (auth_option_pattern[i], key, 0) == 0) {
+                        dict_set ((dict_t *)xl_dict, key, value);
+                        break;
+                }
+        }
 
-        if (fnmatch ( auth_option_pattern[1], key, 0) != 0)
-                dict_set ((dict_t *)xl_dict, key, (value));
+        return 0;
 }
 
 
@@ -715,18 +637,31 @@ server_init_grace_timer (xlator_t *this, dict_t *options,
 {
         int32_t   ret            = -1;
         int32_t   grace_timeout  = -1;
+        char     *lk_heal        = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", this, out);
         GF_VALIDATE_OR_GOTO (this->name, options, out);
         GF_VALIDATE_OR_GOTO (this->name, conf, out);
 
+        conf->lk_heal = _gf_false;
+
+        ret = dict_get_str (options, "lk-heal", &lk_heal);
+        if (!ret)
+                gf_string2boolean (lk_heal, &conf->lk_heal);
+
+        gf_log (this->name, GF_LOG_DEBUG, "lk-heal = %s",
+                (conf->lk_heal) ? "on" : "off");
+
         ret = dict_get_int32 (options, "grace-timeout", &grace_timeout);
         if (!ret)
-                conf->grace_tv.tv_sec = grace_timeout;
+                conf->grace_ts.tv_sec = grace_timeout;
         else
-                conf->grace_tv.tv_sec = 10;
+                conf->grace_ts.tv_sec = 10;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Server grace timeout "
+                "value = %"PRIu64, conf->grace_ts.tv_sec);
 
-        conf->grace_tv.tv_usec  = 0;
+        conf->grace_ts.tv_nsec  = 0;
 
         ret = 0;
 out:
@@ -764,22 +699,15 @@ reconfigure (xlator_t *this, dict_t *options)
                         gf_log (this->name, GF_LOG_WARNING,
                                 "'trace' takes on only boolean values. "
                                 "Neglecting option");
-			ret = -1;
+                        ret = -1;
                         goto out;
-		}
-		conf->trace = trace;
-		gf_log (this->name, GF_LOG_TRACE, "Reconfigured trace"
-			" to %d", conf->trace);
+                }
+                conf->trace = trace;
+                gf_log (this->name, GF_LOG_TRACE, "Reconfigured trace"
+                        " to %d", conf->trace);
 
-	}
+        }
 
-        /*ret = dict_get_str (options, "statedump-path", &statedump_path);
-        if (!ret) {
-                gf_path_strip_trailing_slashes (statedump_path);
-                if (this->ctx->statedump_path)
-                        GF_FREE (this->ctx->statedump_path);
-                this->ctx->statedump_path = gf_strdup (statedump_path);
-        }*/
         GF_OPTION_RECONF ("statedump-path", statedump_path,
                           options, path, out);
         if (!statedump_path) {
@@ -789,8 +717,7 @@ reconfigure (xlator_t *this, dict_t *options)
                 goto out;
         }
         gf_path_strip_trailing_slashes (statedump_path);
-        if (this->ctx->statedump_path)
-                GF_FREE (this->ctx->statedump_path);
+        GF_FREE (this->ctx->statedump_path);
         this->ctx->statedump_path = gf_strdup (statedump_path);
 
         if (!conf->auth_modules)
@@ -818,6 +745,16 @@ reconfigure (xlator_t *this, dict_t *options)
         }
 
         (void) rpcsvc_set_allow_insecure (rpc_conf, options);
+        (void) rpcsvc_set_root_squash (rpc_conf, options);
+
+        ret = rpcsvc_set_outstanding_rpc_limit (rpc_conf, options,
+                                         RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to reconfigure outstanding-rpc-limit");
+                goto out;
+        }
+
         list_for_each_entry (listeners, &(rpc_conf->listeners), list) {
                 if (listeners->trans != NULL) {
                         if (listeners->trans->reconfigure )
@@ -834,6 +771,26 @@ out:
         return ret;
 }
 
+static int32_t
+client_destroy_cbk (xlator_t *this, client_t *client)
+{
+        void         *tmp = NULL;
+        server_ctx_t *ctx = NULL;
+
+        client_ctx_del (client, this, &tmp);
+
+        ctx = tmp;
+
+        if (ctx == NULL)
+                return 0;
+
+        gf_fd_fdtable_destroy (ctx->fdtable);
+        LOCK_DESTROY (&ctx->fdtable_lock);
+        GF_FREE (ctx);
+
+        return 0;
+}
+
 int
 init (xlator_t *this)
 {
@@ -860,7 +817,6 @@ init (xlator_t *this)
 
         GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        INIT_LIST_HEAD (&conf->conns);
         INIT_LIST_HEAD (&conf->xprt_list);
         pthread_mutex_init (&conf->mutex, NULL);
 
@@ -912,12 +868,20 @@ init (xlator_t *this)
         /* RPC related */
         conf->rpc = rpcsvc_init (this, this->ctx, this->options, 0);
         if (conf->rpc == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
+                gf_log (this->name, GF_LOG_ERROR,
                         "creation of rpcsvc failed");
                 ret = -1;
                 goto out;
         }
 
+        ret = rpcsvc_set_outstanding_rpc_limit (conf->rpc, this->options,
+                                         RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to configure outstanding-rpc-limit");
+                goto out;
+        }
+
         ret = rpcsvc_create_listeners (conf->rpc, this->options,
                                        this->name);
         if (ret < 1) {
@@ -934,14 +898,14 @@ init (xlator_t *this)
                 goto out;
         }
 
-        glusterfs3_1_fop_prog.options = this->options;
-        ret = rpcsvc_program_register (conf->rpc, &glusterfs3_1_fop_prog);
+        glusterfs3_3_fop_prog.options = this->options;
+        ret = rpcsvc_program_register (conf->rpc, &glusterfs3_3_fop_prog);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "registration of program (name:%s, prognum:%d, "
-                        "progver:%d) failed", glusterfs3_1_fop_prog.progname,
-                        glusterfs3_1_fop_prog.prognum,
-                        glusterfs3_1_fop_prog.progver);
+                        "progver:%d) failed", glusterfs3_3_fop_prog.progname,
+                        glusterfs3_3_fop_prog.prognum,
+                        glusterfs3_3_fop_prog.progver);
                 goto out;
         }
 
@@ -953,7 +917,7 @@ init (xlator_t *this)
                         "progver:%d) failed", gluster_handshake_prog.progname,
                         gluster_handshake_prog.prognum,
                         gluster_handshake_prog.progver);
-                rpcsvc_program_unregister (conf->rpc, &glusterfs3_1_fop_prog);
+                rpcsvc_program_unregister (conf->rpc, &glusterfs3_3_fop_prog);
                 goto out;
         }
 
@@ -1045,19 +1009,19 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 }
 
 
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
 
 struct xlator_cbks cbks = {
+        .client_destroy = client_destroy_cbk,
 };
 
 struct xlator_dumpops dumpops = {
         .priv           = server_priv,
-        .fd             = server_fd,
-        .inode          = server_inode,
+        .fd             = gf_client_dump_fdtables,
+        .inode          = gf_client_dump_inodes,
         .priv_to_dict   = server_priv_to_dict,
-        .fd_to_dict     = server_fd_to_dict,
-        .inode_to_dict  = server_inode_to_dict,
+        .fd_to_dict     = gf_client_dump_fdtables_to_dict,
+        .inode_to_dict  = gf_client_dump_inodes_to_dict,
 };
 
 
@@ -1085,6 +1049,8 @@ struct volume_options options[] = {
           .min   = 0,
           .max   = (1 * GF_UNIT_MB),
           .default_value = "16384",
+          .description = "Specifies the maximum megabytes of memory to be "
+          "used in the inode cache."
         },
         { .key   = {"verify-volfile-checksum"},
           .type  = GF_OPTION_TYPE_BOOL
@@ -1099,17 +1065,69 @@ struct volume_options options[] = {
         { .key   = {"rpc-auth-allow-insecure"},
           .type  = GF_OPTION_TYPE_BOOL,
         },
+        { .key   = {"root-squash"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Map requests from uid/gid 0 to the anonymous "
+                         "uid/gid. Note that this does not apply to any other "
+                         "uids or gids that might be equally sensitive, such "
+                         "as user bin or group staff."
+        },
+        { .key           = {"anonuid"},
+          .type          = GF_OPTION_TYPE_INT,
+          .default_value = "65534", /* RPC_NOBODY_UID */
+          .min           = 0,
+          .max           = (uint32_t) -1,
+          .description   = "value of the uid used for the anonymous "
+                           "user/nfsnobody when root-squash is enabled."
+        },
+        { .key           = {"anongid"},
+          .type          = GF_OPTION_TYPE_INT,
+          .default_value = "65534", /* RPC_NOBODY_GID */
+          .min           = 0,
+          .max           = (uint32_t) -1,
+          .description   = "value of the gid used for the anonymous "
+                           "user/nfsnobody when root-squash is enabled."
+        },
         { .key           = {"statedump-path"},
           .type          = GF_OPTION_TYPE_PATH,
-          .default_value = "/tmp"
+          .default_value = DEFAULT_VAR_RUN_DIRECTORY,
+          .description = "Specifies directory in which gluster should save its"
+                         " statedumps. By default it is the /tmp directory"
+        },
+        { .key   = {"lk-heal"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
         },
         {.key  = {"grace-timeout"},
          .type = GF_OPTION_TYPE_INT,
+         .min  = 10,
+         .max  = 1800,
         },
         {.key  = {"tcp-window-size"},
          .type = GF_OPTION_TYPE_SIZET,
          .min  = GF_MIN_SOCKET_WINDOW_SIZE,
-         .max  = GF_MAX_SOCKET_WINDOW_SIZE
+         .max  = GF_MAX_SOCKET_WINDOW_SIZE,
+         .description = "Specifies the window size for tcp socket."
+        },
+
+        /*  The following two options are defined in addr.c, redifined here *
+         * for the sake of validation during volume set from cli            */
+
+        { .key   = {"auth.addr.*.allow"},
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .description = "Allow a comma separated list of addresses and/or "
+                         "hostnames to connect to the server. Option "
+                         "auth.reject overrides this option. By default, all "
+                         "connections are allowed."
+        },
+        { .key   = {"auth.addr.*.reject"},
+          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+          .description = "Reject a comma separated list of addresses and/or "
+                         "hostnames to connect to the server. This option "
+                         "overrides the auth.allow option. By default, all"
+                         " connections are allowed."
         },
+
         { .key   = {NULL} },
 };
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index c54454e3e..43e84921c 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _SERVER_H
@@ -22,6 +13,7 @@
 
 #include <pthread.h>
 
+#include "fd.h"
 #include "rpcsvc.h"
 
 #include "fd.h"
@@ -29,60 +21,19 @@
 #include "server-mem-types.h"
 #include "glusterfs3.h"
 #include "timer.h"
+#include "client_t.h"
 
 #define DEFAULT_BLOCK_SIZE         4194304   /* 4MB */
 #define DEFAULT_VOLUME_FILE_PATH   CONFDIR "/glusterfs.vol"
 #define GF_MAX_SOCKET_WINDOW_SIZE  (1 * GF_UNIT_MB)
 #define GF_MIN_SOCKET_WINDOW_SIZE  (0)
 
-typedef struct _server_state server_state_t;
-
-struct _locker {
-        struct list_head  lockers;
-        char             *volume;
-        loc_t             loc;
-        fd_t             *fd;
-        gf_lkowner_t      owner;
-        pid_t             pid;
-};
-
-struct _lock_table {
-        struct list_head  inodelk_lockers;
-        struct list_head  entrylk_lockers;
-        gf_lock_t         lock;
-        size_t            count;
-};
-
-/* private structure per connection (transport object)
- * used as transport_t->xl_private
- */
-struct _server_connection {
-        struct list_head    list;
-        char               *id;
-        int                 ref;
-        pthread_mutex_t     lock;
-        fdtable_t          *fdtable;
-        struct _lock_table *ltable;
-        gf_timer_t         *timer;
-        xlator_t           *bound_xl;
-        xlator_t           *this;
-        uint32_t           lk_version;
-};
-
-typedef struct _server_connection server_connection_t;
-
-
-server_connection_t *
-server_connection_get (xlator_t *this, const char *id);
-
-server_connection_t*
-server_conn_unref (server_connection_t *conn);
-
-server_connection_t*
-server_conn_ref (server_connection_t *conn);
+typedef enum {
+        INTERNAL_LOCKS = 1,
+        POSIX_LOCKS = 2,
+} server_lock_flags_t;
 
-int
-server_connection_cleanup (xlator_t *this, server_connection_t *conn);
+typedef struct _server_state server_state_t;
 
 int server_null (rpcsvc_request_t *req);
 
@@ -98,12 +49,13 @@ struct server_conf {
         int                     inode_lru_limit;
         gf_boolean_t            verify_volfile;
         gf_boolean_t            trace;
+        gf_boolean_t            lk_heal; /* If true means lock self
+                                            heal is on else off. */
         char                   *conf_dir;
         struct _volfile_ctx    *volfile;
-        struct timeval          grace_tv;
+        struct timespec         grace_ts;
         dict_t                 *auth_modules;
         pthread_mutex_t         mutex;
-        struct list_head        conns;
         struct list_head        xprt_list;
 };
 typedef struct server_conf server_conf_t;
@@ -125,7 +77,7 @@ struct resolve_comp {
 
 typedef struct {
         server_resolve_type_t  type;
-        uint64_t               fd_no;
+        int64_t               fd_no;
         u_char                 gfid[16];
         u_char                 pargfid[16];
         char                  *path;
@@ -142,11 +94,10 @@ int
 resolve_and_resume (call_frame_t *frame, server_resume_fn_t fn);
 
 struct _server_state {
-        server_connection_t  *conn;
-        rpc_transport_t      *xprt;
-        inode_table_t        *itable;
+        rpc_transport_t  *xprt;
+        inode_table_t    *itable;
 
-        server_resume_fn_t    resume_fn;
+        server_resume_fn_t resume_fn;
 
         loc_t             loc;
         loc_t             loc2;
@@ -182,15 +133,28 @@ struct _server_state {
         int               mask;
         char              is_revalidate;
         dict_t           *dict;
-        struct gf_flock      flock;
+        struct gf_flock   flock;
         const char       *volume;
         dir_entry_t      *entry;
+
+        dict_t           *xdata;
+        mode_t            umask;
 };
 
+
 extern struct rpcsvc_program gluster_handshake_prog;
-extern struct rpcsvc_program glusterfs3_1_fop_prog;
+extern struct rpcsvc_program glusterfs3_3_fop_prog;
 extern struct rpcsvc_program gluster_ping_prog;
 
+
+typedef struct _server_ctx {
+        gf_lock_t            fdtable_lock;
+        fdtable_t           *fdtable;
+        struct _gf_timer    *grace_timer;
+        uint32_t             lk_version;
+} server_ctx_t;
+
+
 int
 server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
                      struct iovec *payload, int payloadcount,
diff --git a/xlators/protocol/server/src/server3_1-fops.c b/xlators/protocol/server/src/server3_1-fops.c
deleted file mode 100644
index 3dae07b03..000000000
--- a/xlators/protocol/server/src/server3_1-fops.c
+++ /dev/null
@@ -1,5007 +0,0 @@
-/*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "server.h"
-#include "server-helpers.h"
-#include "glusterfs3-xdr.h"
-#include "glusterfs3.h"
-#include "compat-errno.h"
-
-#include "md5.h"
-#include "xdr-nfs3.h"
-
-
-/* Callback function section */
-int
-server_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *buf)
-{
-        gfs3_statfs_rsp   rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        if (op_ret >= 0) {
-                gf_statfs_from_statfs (&rsp.statfs, buf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": STATFS %"PRId32" (%s)",
-                        frame->root->unique, op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_statfs_rsp);
-
-        return 0;
-}
-
-int
-server_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *stbuf, dict_t *dict,
-                   struct iatt *postparent)
-{
-        rpcsvc_request_t *req        = NULL;
-        server_state_t   *state      = NULL;
-        inode_t          *root_inode = NULL;
-        inode_t          *link_inode = NULL;
-        loc_t             fresh_loc  = {0,};
-        gfs3_lookup_rsp   rsp        = {0,};
-        uuid_t            rootgfid   = {0,};
-
-        state = CALL_STATE(frame);
-
-        req = frame->local;
-
-        if (state->is_revalidate == 1 && op_ret == -1) {
-                state->is_revalidate = 2;
-                loc_copy (&fresh_loc, &state->loc);
-                inode_unref (fresh_loc.inode);
-                fresh_loc.inode = inode_new (state->itable);
-
-                STACK_WIND (frame, server_lookup_cbk, BOUND_XL (frame),
-                            BOUND_XL (frame)->fops->lookup,
-                            &fresh_loc, state->dict);
-
-                loc_wipe (&fresh_loc);
-                return 0;
-        }
-
-        if ((op_ret >= 0) && dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-
-        gf_stat_from_iatt (&rsp.postparent, postparent);
-
-        if (op_ret == 0) {
-                root_inode = BOUND_XL(frame)->itable->root;
-                if (inode == root_inode) {
-                        /* we just looked up root ("/") */
-                        stbuf->ia_ino = 1;
-                        rootgfid[15]  = 1;
-                        uuid_copy (stbuf->ia_gfid, rootgfid);
-                        if (inode->ia_type == 0)
-                                inode->ia_type = stbuf->ia_type;
-                }
-
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-
-                if (!__is_root_gfid (inode->gfid)) {
-                        link_inode = inode_link (inode, state->loc.parent,
-                                                 state->loc.name, stbuf);
-                        inode_lookup (link_inode);
-                        inode_unref (link_inode);
-                }
-        } else {
-                if (state->is_revalidate && op_errno == ENOENT) {
-                        if (!__is_root_gfid (state->loc.inode->gfid)) {
-                                inode_unlink (state->loc.inode,
-                                              state->loc.parent,
-                                              state->loc.name);
-                        }
-                }
-        }
-out:
-        rsp.op_ret   = op_ret;
-        rsp.op_errno = gf_errno_to_error (op_errno);
-
-        if (op_ret) {
-                gf_log (this->name,
-                        ((op_errno == ENOENT) ? GF_LOG_TRACE : GF_LOG_INFO),
-                        "%"PRId64": LOOKUP %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_lookup_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-
-int
-server_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        gfs3_lk_rsp       rsp   = {0,};
-        rpcsvc_request_t *req   = NULL;
-        server_state_t   *state = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                switch (lock->l_type) {
-                case F_RDLCK:
-                        lock->l_type = GF_LK_F_RDLCK;
-                        break;
-                case F_WRLCK:
-                        lock->l_type = GF_LK_F_WRLCK;
-                        break;
-                case F_UNLCK:
-                        lock->l_type = GF_LK_F_UNLCK;
-                        break;
-                default:
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unknown lock type: %"PRId32"!", lock->l_type);
-                        break;
-                }
-
-                gf_proto_flock_from_flock (&rsp.flock, lock);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": LK %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_lk_rsp);
-
-        return 0;
-}
-
-
-int
-server_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp        rsp   = {0,};
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        conn  = SERVER_CONNECTION(frame);
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                if (state->flock.l_type == F_UNLCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, &frame->root->lk_owner,
-                                       GF_FOP_INODELK);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid,
-                                       &frame->root->lk_owner,
-                                       GF_FOP_INODELK);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": INODELK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-
-int
-server_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp        rsp   = {0,};
-        server_state_t      *state = NULL;
-        server_connection_t *conn  = NULL;
-        rpcsvc_request_t    *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        conn = SERVER_CONNECTION(frame);
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                if (state->flock.l_type == F_UNLCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       NULL, state->fd,
-                                       &frame->root->lk_owner,  GF_FOP_INODELK);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       NULL, state->fd,
-                                       frame->root->pid,
-                                       &frame->root->lk_owner, GF_FOP_INODELK);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FINODELK %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-        gf_common_rsp        rsp   = {0,};
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        conn  = SERVER_CONNECTION(frame);
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                if (state->cmd == ENTRYLK_UNLOCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, &frame->root->lk_owner,
-                                       GF_FOP_ENTRYLK);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       &state->loc, NULL, frame->root->pid,
-                                       &frame->root->lk_owner,
-                                       GF_FOP_ENTRYLK);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": ENTRYLK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-        return 0;
-}
-
-
-int
-server_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp        rsp   = {0,};
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        conn  = SERVER_CONNECTION(frame);
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                if (state->cmd == ENTRYLK_UNLOCK)
-                        gf_del_locker (conn->ltable, state->volume,
-                                       NULL, state->fd, &frame->root->lk_owner,
-                                       GF_FOP_ENTRYLK);
-                else
-                        gf_add_locker (conn->ltable, state->volume,
-                                       NULL, state->fd, frame->root->pid,
-                                       &frame->root->lk_owner, GF_FOP_ENTRYLK);
-        } else if ((op_errno != ENOSYS) && (op_errno != EAGAIN)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FENTRYLK %"PRId64" (%s) "
-                        " ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-
-int
-server_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t   *state = NULL;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-        if (op_ret)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": ACCESS %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gfs3_rmdir_rsp    rsp    = {0,};
-        server_state_t   *state  = NULL;
-        inode_t          *parent = NULL;
-        rpcsvc_request_t *req    = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-                parent = inode_parent (state->loc.inode, 0, NULL);
-                if (parent)
-                        inode_unref (parent);
-                else
-                        inode_forget (state->loc.inode, 0);
-
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": RMDIR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_rmdir_rsp);
-
-        return 0;
-}
-
-int
-server_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gfs3_mkdir_rsp    rsp        = {0,};
-        server_state_t   *state      = NULL;
-        inode_t          *link_inode = NULL;
-        rpcsvc_request_t *req        = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": MKDIR %s  ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_mkdir_rsp);
-
-        return 0;
-}
-
-int
-server_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        gfs3_mknod_rsp    rsp        = {0,};
-        server_state_t   *state      = NULL;
-        inode_t          *link_inode = NULL;
-        rpcsvc_request_t *req        = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": MKNOD %s ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_mknod_rsp);
-
-
-        return 0;
-}
-
-int
-server_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSYNCDIR %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        gfs3_readdir_rsp  rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-        int               ret   = 0;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        if (op_ret > 0) {
-                ret = serialize_rsp_dirent (entries, &rsp);
-                if (ret == -1) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-        } else {
-                /* (op_ret == 0) is valid, and means EOF, don't log for that */
-                gf_log (this->name, (op_ret) ? GF_LOG_INFO : GF_LOG_TRACE,
-                        "%"PRId64": READDIR %"PRId64" (%s) ==> %"PRId32
-                        " (%s)", frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-unwind:
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_readdir_rsp);
-
-        readdir_rsp_cleanup (&rsp);
-
-        return 0;
-}
-
-
-int
-server_releasedir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-        gfs3_opendir_rsp     rsp   = {0,};
-        uint64_t             fd_no = 0;
-
-        conn = SERVER_CONNECTION (frame);
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd); // on behalf of the client
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": OPENDIR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        req           = frame->local;
-
-        rsp.fd        = fd_no;
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_opendir_rsp);
-
-        return 0;
-}
-
-int
-server_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = frame->local;
-        state = CALL_STATE(frame);
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": REMOVEXATTR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req   = frame->local;
-        state = CALL_STATE(frame);
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FREMOVEXATTR (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, ((state->fd->inode) ?
-                                              uuid_utoa (state->fd->inode->gfid) :
-                                              "--"),
-                        op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gfs3_getxattr_rsp  rsp   = {0,};
-        rpcsvc_request_t  *req   = NULL;
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-out:
-        req               = frame->local;
-
-        rsp.op_ret        = op_ret;
-        rsp.op_errno      = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": GETXATTR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->name, op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_getxattr_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-
-int
-server_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gfs3_fgetxattr_rsp  rsp   = {0,};
-        server_state_t     *state = NULL;
-        rpcsvc_request_t   *req   = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-
-out:
-        req               = frame->local;
-
-        rsp.op_ret        = op_ret;
-        rsp.op_errno      = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FGETXATTR %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->name, op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fgetxattr_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-int
-server_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": SETXATTR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-
-int
-server_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp = {0,};
-        rpcsvc_request_t *req = NULL;
-        server_state_t      *state = NULL;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSETXATTR %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-        return 0;
-}
-
-int
-server_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent)
-{
-        gfs3_rename_rsp   rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                stbuf->ia_type = state->loc.inode->ia_type;
-
-                /* TODO: log gfid of the inodes */
-                gf_log (state->conn->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": RENAME_CBK  %s ==> %s",
-                        frame->root->unique, state->loc.name, state->loc2.name);
-
-                inode_rename (state->itable,
-                              state->loc.parent, state->loc.name,
-                              state->loc2.parent, state->loc2.name,
-                              state->loc.inode, stbuf);
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-
-                gf_stat_from_iatt (&rsp.preoldparent, preoldparent);
-                gf_stat_from_iatt (&rsp.postoldparent, postoldparent);
-
-                gf_stat_from_iatt (&rsp.prenewparent, prenewparent);
-                gf_stat_from_iatt (&rsp.postnewparent, postnewparent);
-        }
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": RENAME %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_rename_rsp);
-
-        return 0;
-}
-
-int
-server_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        gfs3_unlink_rsp   rsp    = {0,};
-        server_state_t   *state  = NULL;
-        inode_t          *parent = NULL;
-        rpcsvc_request_t *req    = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                /* TODO: log gfid of the inodes */
-                gf_log (state->conn->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": UNLINK_CBK %s",
-                        frame->root->unique, state->loc.name);
-
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-
-                parent = inode_parent (state->loc.inode, 0, NULL);
-                if (parent)
-                        inode_unref (parent);
-                else
-                        inode_forget (state->loc.inode, 0);
-
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": UNLINK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_unlink_rsp);
-
-        return 0;
-}
-
-int
-server_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-        gfs3_symlink_rsp  rsp        = {0,};
-        server_state_t   *state      = NULL;
-        inode_t          *link_inode = NULL;
-        rpcsvc_request_t *req        = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": SYMLINK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_symlink_rsp);
-
-        return 0;
-}
-
-
-int
-server_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent)
-{
-        gfs3_link_rsp     rsp        = {0,};
-        server_state_t   *state      = NULL;
-        inode_t          *link_inode = NULL;
-        rpcsvc_request_t *req        = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-
-                link_inode = inode_link (inode, state->loc2.parent,
-                                         state->loc2.name, stbuf);
-                inode_unref (link_inode);
-        }
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": LINK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_link_rsp);
-
-        return 0;
-}
-
-int
-server_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
-{
-        gfs3_truncate_rsp  rsp   = {0,};
-        server_state_t    *state = NULL;
-        rpcsvc_request_t  *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.prestat, prebuf);
-                gf_stat_from_iatt (&rsp.poststat, postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": TRUNCATE %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_truncate_rsp);
-
-        return 0;
-}
-
-int
-server_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
-{
-        gfs3_fstat_rsp    rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSTAT %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fstat_rsp);
-
-        return 0;
-}
-
-int
-server_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-        gfs3_ftruncate_rsp  rsp   = {0};
-        server_state_t     *state = NULL;
-        rpcsvc_request_t   *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.prestat, prebuf);
-                gf_stat_from_iatt (&rsp.poststat, postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FTRUNCATE %"PRId64" (%s)==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_ftruncate_rsp);
-
-        return 0;
-}
-
-int
-server_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        gf_common_rsp     rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FLUSH %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-
-
-        return 0;
-}
-
-int
-server_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-        gfs3_fsync_rsp    rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&(rsp.prestat), prebuf);
-                gf_stat_from_iatt (&(rsp.poststat), postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSYNC %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fsync_rsp);
-
-        return 0;
-}
-
-int
-server_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-        gfs3_write_rsp    rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.prestat, prebuf);
-                gf_stat_from_iatt (&rsp.poststat, postbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": WRITEV %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_write_rsp);
-
-        return 0;
-}
-
-
-int
-server_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count,
-                  struct iatt *stbuf, struct iobref *iobref)
-{
-        gfs3_read_rsp     rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state = CALL_STATE(frame);
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                rsp.size = op_ret;
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": READV %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, vector, count, iobref,
-                             (xdrproc_t)xdr_gfs3_read_rsp);
-
-        return 0;
-}
-
-int
-server_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
-{
-        gfs3_rchecksum_rsp  rsp = {0,};
-        rpcsvc_request_t   *req = NULL;
-        server_state_t      *state = NULL;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        if (op_ret >= 0) {
-                rsp.weak_checksum = weak_checksum;
-
-                rsp.strong_checksum.strong_checksum_val = (char *)strong_checksum;
-                rsp.strong_checksum.strong_checksum_len = MD5_DIGEST_LEN;
-        }
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": RCHECKSUM %"PRId64" (%s)==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_rchecksum_rsp);
-
-        return 0;
-}
-
-
-int
-server_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        server_connection_t *conn  = NULL;
-        server_state_t      *state = NULL;
-        rpcsvc_request_t    *req   = NULL;
-        uint64_t             fd_no = 0;
-        gfs3_open_rsp        rsp   = {0,};
-
-        conn = SERVER_CONNECTION (frame);
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                fd_bind (fd);
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": OPEN %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        req           = frame->local;
-
-        rsp.fd        = fd_no;
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_open_rsp);
-        return 0;
-}
-
-
-int
-server_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                   struct iatt *preparent, struct iatt *postparent)
-{
-        server_connection_t *conn       = NULL;
-        server_state_t      *state      = NULL;
-        inode_t             *link_inode = NULL;
-        rpcsvc_request_t    *req        = NULL;
-        uint64_t             fd_no      = 0;
-        gfs3_create_rsp      rsp        = {0,};
-
-        conn = SERVER_CONNECTION (frame);
-        state = CALL_STATE (frame);
-
-        if (op_ret >= 0) {
-                /* TODO: log gfid too */
-                gf_log (state->conn->bound_xl->name, GF_LOG_TRACE,
-                        "%"PRId64": CREATE %s (%s)",
-                        frame->root->unique, state->loc.name,
-                        uuid_utoa (inode->gfid));
-
-                link_inode = inode_link (inode, state->loc.parent,
-                                         state->loc.name, stbuf);
-
-                if (!link_inode) {
-                        op_ret = -1;
-                        op_errno = ENOENT;
-                        goto out;
-                }
-
-                if (link_inode != inode) {
-                        /*
-                          VERY racy code (if used anywhere else)
-                          -- don't do this without understanding
-                        */
-
-                        inode_unref (fd->inode);
-                        fd->inode = inode_ref (link_inode);
-                }
-
-                inode_lookup (link_inode);
-                inode_unref (link_inode);
-
-                fd_bind (fd);
-
-                fd_no = gf_fd_unused_get (conn->fdtable, fd);
-                fd_ref (fd);
-
-                if ((fd_no < 0) || (fd == 0)) {
-                        op_ret = fd_no;
-                        op_errno = errno;
-                }
-
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-                gf_stat_from_iatt (&rsp.preparent, preparent);
-                gf_stat_from_iatt (&rsp.postparent, postparent);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": CREATE %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-out:
-        req           = frame->local;
-
-        rsp.fd        = fd_no;
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_create_rsp);
-
-        return 0;
-}
-
-int
-server_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, const char *buf,
-                     struct iatt *stbuf)
-{
-        gfs3_readlink_rsp  rsp   = {0,};
-        server_state_t    *state = NULL;
-        rpcsvc_request_t  *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-
-        state  = CALL_STATE(frame);
-
-        if (op_ret >= 0) {
-                gf_stat_from_iatt (&rsp.buf, stbuf);
-                rsp.path = (char *)buf;
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": READLINK %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        if (!rsp.path)
-                rsp.path = "";
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_readlink_rsp);
-
-        return 0;
-}
-
-int
-server_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *stbuf)
-{
-        gfs3_stat_rsp     rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state  = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.stat, stbuf);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": STAT %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_stat_rsp);
-
-        return 0;
-}
-
-
-int
-server_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost)
-{
-        gfs3_setattr_rsp  rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        state  = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.statpre, statpre);
-                gf_stat_from_iatt (&rsp.statpost, statpost);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": SETATTR %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-        }
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_setattr_rsp);
-
-        return 0;
-}
-
-int
-server_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *statpre, struct iatt *statpost)
-{
-        gfs3_fsetattr_rsp  rsp   = {0,};
-        server_state_t    *state = NULL;
-        rpcsvc_request_t  *req   = NULL;
-
-        state  = CALL_STATE (frame);
-
-        if (op_ret == 0) {
-                gf_stat_from_iatt (&rsp.statpre, statpre);
-                gf_stat_from_iatt (&rsp.statpost, statpost);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FSETATTR %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-        req           = frame->local;
-
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fsetattr_rsp);
-
-        return 0;
-}
-
-
-int
-server_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gfs3_xattrop_rsp  rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": XATTROP %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-                goto out;
-        }
-
-        if ((op_ret >= 0) && dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-out:
-        req               = frame->local;
-
-        rsp.op_ret        = op_ret;
-        rsp.op_errno      = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": XATTROP %s (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->loc.path,
-                        state->loc.inode ? uuid_utoa (state->loc.inode->gfid) :
-                        "--", op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_xattrop_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-
-int
-server_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        gfs3_xattrop_rsp  rsp   = {0,};
-        server_state_t   *state = NULL;
-        rpcsvc_request_t *req   = NULL;
-
-        state = CALL_STATE(frame);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%"PRId64": FXATTROP %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-                goto out;
-        }
-
-        if ((op_ret >= 0) && dict) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, dict,
-                                            (&rsp.dict.dict_val),
-                                            rsp.dict.dict_len,
-                                            op_errno, out);
-        }
-out:
-        req               = frame->local;
-
-        rsp.op_ret        = op_ret;
-        rsp.op_errno      = gf_errno_to_error (op_errno);
-
-        if (op_ret == -1)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": FXATTROP %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_fxattrop_rsp);
-
-        if (rsp.dict.dict_val)
-                GF_FREE (rsp.dict.dict_val);
-
-        return 0;
-}
-
-
-int
-server_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        gfs3_readdirp_rsp  rsp   = {0,};
-        server_state_t    *state = NULL;
-        rpcsvc_request_t  *req   = NULL;
-        int                ret   = 0;
-
-        req           = frame->local;
-
-        state = CALL_STATE(frame);
-        if (op_ret > 0) {
-                ret = serialize_rsp_direntp (entries, &rsp);
-                if (ret == -1) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                /* TODO: need more clear thoughts before calling this function. */
-                /* gf_link_inodes_from_dirent (this, state->fd->inode, entries); */
-
-        } else {
-                /* (op_ret == 0) is valid, and means EOF, don't log for that */
-                gf_log (this->name, (op_ret) ? GF_LOG_INFO : GF_LOG_TRACE,
-                        "%"PRId64": READDIRP %"PRId64" (%s) ==> %"PRId32" (%s)",
-                        frame->root->unique, state->resolve.fd_no,
-                        state->fd ? uuid_utoa (state->fd->inode->gfid) : "--",
-                        op_ret, strerror (op_errno));
-        }
-
-out:
-        rsp.op_ret    = op_ret;
-        rsp.op_errno  = gf_errno_to_error (op_errno);
-
-        server_submit_reply (frame, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gfs3_readdirp_rsp);
-
-        readdirp_rsp_cleanup (&rsp);
-
-        return 0;
-}
-
-/* Resume function section */
-
-int
-server_rchecksum_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state    = NULL;
-        int             op_ret   = 0;
-        int             op_errno = EINVAL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        STACK_WIND (frame, server_rchecksum_cbk, bound_xl,
-                    bound_xl->fops->rchecksum, state->fd,
-                    state->offset, state->size);
-
-        return 0;
-err:
-        server_rchecksum_cbk (frame, NULL, frame->this, op_ret, op_errno, 0, NULL);
-
-        return 0;
-
-}
-
-int
-server_lk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_lk_cbk, bound_xl, bound_xl->fops->lk,
-                    state->fd, state->cmd, &state->flock);
-
-        return 0;
-
-err:
-        server_lk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                       state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_rename_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        if (state->resolve2.op_ret != 0) {
-                op_ret   = state->resolve2.op_ret;
-                op_errno = state->resolve2.op_errno;
-                goto err;
-        }
-
-        STACK_WIND (frame, server_rename_cbk,
-                    bound_xl, bound_xl->fops->rename,
-                    &state->loc, &state->loc2);
-        return 0;
-err:
-        server_rename_cbk (frame, NULL, frame->this, op_ret, op_errno,
-                           NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_link_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-        int             op_ret = 0;
-        int             op_errno = 0;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0) {
-                op_ret   = state->resolve.op_ret;
-                op_errno = state->resolve.op_errno;
-                goto err;
-        }
-
-        if (state->resolve2.op_ret != 0) {
-                op_ret   = state->resolve2.op_ret;
-                op_errno = state->resolve2.op_errno;
-                goto err;
-        }
-
-        state->loc2.inode = inode_ref (state->loc.inode);
-
-        STACK_WIND (frame, server_link_cbk, bound_xl, bound_xl->fops->link,
-                    &state->loc, &state->loc2);
-
-        return 0;
-err:
-        server_link_cbk (frame, NULL, frame->this, op_ret, op_errno,
-                         NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-int
-server_symlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_symlink_cbk,
-                    bound_xl, bound_xl->fops->symlink,
-                    state->name, &state->loc, state->params);
-
-        return 0;
-err:
-        server_symlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_access_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_access_cbk,
-                    bound_xl, bound_xl->fops->access,
-                    &state->loc, state->mask);
-        return 0;
-err:
-        server_access_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno);
-        return 0;
-}
-
-int
-server_fentrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fentrylk_cbk, bound_xl,
-                    bound_xl->fops->fentrylk,
-                    state->volume, state->fd, state->name,
-                    state->cmd, state->type);
-
-        return 0;
-err:
-        server_fentrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-        return 0;
-}
-
-
-int
-server_entrylk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_entrylk_cbk,
-                    bound_xl, bound_xl->fops->entrylk,
-                    state->volume, &state->loc, state->name,
-                    state->cmd, state->type);
-        return 0;
-err:
-        server_entrylk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno);
-        return 0;
-}
-
-
-int
-server_finodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_finodelk_cbk, bound_xl,
-                    bound_xl->fops->finodelk,
-                    state->volume, state->fd, state->cmd, &state->flock);
-
-        return 0;
-err:
-        server_finodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-
-        return 0;
-}
-
-int
-server_inodelk_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_inodelk_cbk,
-                    bound_xl, bound_xl->fops->inodelk,
-                    state->volume, &state->loc, state->cmd, &state->flock);
-        return 0;
-err:
-        server_inodelk_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno);
-        return 0;
-}
-
-int
-server_rmdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_rmdir_cbk,
-                    bound_xl, bound_xl->fops->rmdir, &state->loc, state->flags);
-        return 0;
-err:
-        server_rmdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-int
-server_mkdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_mkdir_cbk,
-                    bound_xl, bound_xl->fops->mkdir,
-                    &(state->loc), state->mode, state->params);
-
-        return 0;
-err:
-        server_mkdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_mknod_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        STACK_WIND (frame, server_mknod_cbk,
-                    bound_xl, bound_xl->fops->mknod,
-                    &(state->loc), state->mode, state->dev,
-                    state->params);
-
-        return 0;
-err:
-        server_mknod_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_fsyncdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsyncdir_cbk,
-                    bound_xl,
-                    bound_xl->fops->fsyncdir,
-                    state->fd, state->flags);
-        return 0;
-
-err:
-        server_fsyncdir_cbk (frame, NULL, frame->this,
-                             state->resolve.op_ret,
-                             state->resolve.op_errno);
-        return 0;
-}
-
-
-int
-server_readdir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readdir_cbk,
-                    bound_xl,
-                    bound_xl->fops->readdir,
-                    state->fd, state->size, state->offset);
-
-        return 0;
-err:
-        server_readdir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_readdirp_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readdirp_cbk, bound_xl,
-                    bound_xl->fops->readdirp, state->fd, state->size,
-                    state->offset, state->dict);
-
-        return 0;
-err:
-        server_readdirp_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_opendir_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-
-        STACK_WIND (frame, server_opendir_cbk,
-                    bound_xl, bound_xl->fops->opendir,
-                    &state->loc, state->fd);
-        return 0;
-err:
-        server_opendir_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_statfs_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t      *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret !=0)
-                goto err;
-
-        STACK_WIND (frame, server_statfs_cbk,
-                    bound_xl, bound_xl->fops->statfs,
-                    &state->loc);
-        return 0;
-
-err:
-        server_statfs_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_removexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_removexattr_cbk,
-                    bound_xl, bound_xl->fops->removexattr,
-                    &state->loc, state->name);
-        return 0;
-err:
-        server_removexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                                state->resolve.op_errno);
-        return 0;
-}
-
-int
-server_fremovexattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fremovexattr_cbk,
-                    bound_xl, bound_xl->fops->fremovexattr,
-                    state->fd, state->name);
-        return 0;
-err:
-        server_fremovexattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                                 state->resolve.op_errno);
-        return 0;
-}
-
-int
-server_fgetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fgetxattr_cbk,
-                    bound_xl, bound_xl->fops->fgetxattr,
-                    state->fd, state->name);
-        return 0;
-err:
-        server_fgetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_xattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_xattrop_cbk,
-                    bound_xl, bound_xl->fops->xattrop,
-                    &state->loc, state->flags, state->dict);
-        return 0;
-err:
-        server_xattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_fxattrop_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fxattrop_cbk,
-                    bound_xl, bound_xl->fops->fxattrop,
-                    state->fd, state->flags, state->dict);
-        return 0;
-err:
-        server_fxattrop_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_fsetxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setxattr_cbk,
-                    bound_xl, bound_xl->fops->fsetxattr,
-                    state->fd, state->dict, state->flags);
-        return 0;
-err:
-        server_fsetxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno);
-
-        return 0;
-}
-
-int
-server_unlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_unlink_cbk,
-                    bound_xl, bound_xl->fops->unlink,
-                    &state->loc);
-        return 0;
-err:
-        server_unlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-int
-server_truncate_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_truncate_cbk,
-                    bound_xl, bound_xl->fops->truncate,
-                    &state->loc, state->offset);
-        return 0;
-err:
-        server_truncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-
-int
-server_fstat_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t     *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fstat_cbk,
-                    bound_xl, bound_xl->fops->fstat,
-                    state->fd);
-        return 0;
-err:
-        server_fstat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_setxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setxattr_cbk,
-                    bound_xl, bound_xl->fops->setxattr,
-                    &state->loc, state->dict, state->flags);
-        return 0;
-err:
-        server_setxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno);
-
-        return 0;
-}
-
-
-int
-server_getxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_getxattr_cbk,
-                    bound_xl, bound_xl->fops->getxattr,
-                    &state->loc, state->name);
-        return 0;
-err:
-        server_getxattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_ftruncate_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_ftruncate_cbk,
-                    bound_xl, bound_xl->fops->ftruncate,
-                    state->fd, state->offset);
-        return 0;
-err:
-        server_ftruncate_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                              state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-int
-server_flush_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_flush_cbk,
-                    bound_xl, bound_xl->fops->flush, state->fd);
-        return 0;
-err:
-        server_flush_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno);
-
-        return 0;
-}
-
-
-int
-server_fsync_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsync_cbk,
-                    bound_xl, bound_xl->fops->fsync,
-                    state->fd, state->flags);
-        return 0;
-err:
-        server_fsync_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-int
-server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t   *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_writev_cbk,
-                    bound_xl, bound_xl->fops->writev,
-                    state->fd, state->payload_vector, state->payload_count,
-                    state->offset, state->flags, state->iobref);
-
-        return 0;
-err:
-        server_writev_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_readv_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readv_cbk,
-                    bound_xl, bound_xl->fops->readv,
-                    state->fd, state->size, state->offset, state->flags);
-
-        return 0;
-err:
-        server_readv_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                          state->resolve.op_errno, NULL, 0, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_create_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->loc.inode = inode_new (state->itable);
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-        if (!state->fd) {
-                gf_log ("server", GF_LOG_ERROR, "fd creation for the inode %s "
-                        "failed", state->loc.inode?
-                        uuid_utoa (state->loc.inode->gfid):NULL);
-                state->resolve.op_ret = -1;
-                state->resolve.op_errno = ENOMEM;
-                goto err;
-        }
-        state->fd->flags = state->flags;
-
-        STACK_WIND (frame, server_create_cbk,
-                    bound_xl, bound_xl->fops->create,
-                    &(state->loc), state->flags, state->mode,
-                    state->fd, state->params);
-
-        return 0;
-err:
-        server_create_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL, NULL,
-                           NULL, NULL);
-        return 0;
-}
-
-
-int
-server_open_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t  *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        state->fd = fd_create (state->loc.inode, frame->root->pid);
-        state->fd->flags = state->flags;
-
-        STACK_WIND (frame, server_open_cbk,
-                    bound_xl, bound_xl->fops->open,
-                    &state->loc, state->flags, state->fd, 0);
-
-        return 0;
-err:
-        server_open_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                         state->resolve.op_errno, NULL);
-        return 0;
-}
-
-
-int
-server_readlink_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_readlink_cbk,
-                    bound_xl, bound_xl->fops->readlink,
-                    &state->loc, state->size);
-        return 0;
-err:
-        server_readlink_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int
-server_fsetattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_fsetattr_cbk,
-                    bound_xl, bound_xl->fops->fsetattr,
-                    state->fd, &state->stbuf, state->valid);
-        return 0;
-err:
-        server_fsetattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                             state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-int
-server_setattr_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_setattr_cbk,
-                    bound_xl, bound_xl->fops->setattr,
-                    &state->loc, &state->stbuf, state->valid);
-        return 0;
-err:
-        server_setattr_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                            state->resolve.op_errno, NULL, NULL);
-
-        return 0;
-}
-
-
-int
-server_stat_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        STACK_WIND (frame, server_stat_cbk,
-                    bound_xl, bound_xl->fops->stat, &state->loc);
-        return 0;
-err:
-        server_stat_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                         state->resolve.op_errno, NULL);
-        return 0;
-}
-
-int
-server_lookup_resume (call_frame_t *frame, xlator_t *bound_xl)
-{
-        server_state_t    *state = NULL;
-
-        state = CALL_STATE (frame);
-
-        if (state->resolve.op_ret != 0)
-                goto err;
-
-        if (!state->loc.inode)
-                state->loc.inode = inode_new (state->itable);
-        else
-                state->is_revalidate = 1;
-
-        STACK_WIND (frame, server_lookup_cbk,
-                    bound_xl, bound_xl->fops->lookup,
-                    &state->loc, state->dict);
-
-        return 0;
-err:
-        server_lookup_cbk (frame, NULL, frame->this, state->resolve.op_ret,
-                           state->resolve.op_errno, NULL, NULL, NULL, NULL);
-
-        return 0;
-}
-
-
-
-
-/* Fop section */
-
-int
-server_stat (rpcsvc_request_t *req)
-{
-        server_state_t *state                 = NULL;
-        call_frame_t   *frame                 = NULL;
-        gfs3_stat_req   args                  = {{0,},};
-        int             ret                   = -1;
-
-        if (!req)
-                return 0;
-
-        /* Initialize args first, then decode */
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_stat_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_STAT;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_stat_resume);
-out:
-        return ret;
-}
-
-
-int
-server_setattr (rpcsvc_request_t *req)
-{
-        server_state_t   *state                 = NULL;
-        call_frame_t     *frame                 = NULL;
-        gfs3_setattr_req  args                  = {{0,},};
-        int               ret                   = -1;
-
-        if (!req)
-                return 0;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_setattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_SETATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        gf_stat_to_iatt (&args.stbuf, &state->stbuf);
-        state->valid = args.valid;
-
-        ret = 0;
-        resolve_and_resume (frame, server_setattr_resume);
-out:
-        return ret;
-}
-
-
-int
-server_fsetattr (rpcsvc_request_t *req)
-{
-        server_state_t    *state = NULL;
-        call_frame_t      *frame = NULL;
-        gfs3_fsetattr_req  args  = {0,};
-        int                ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fsetattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSETATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = args.fd;
-
-        gf_stat_to_iatt (&args.stbuf, &state->stbuf);
-        state->valid = args.valid;
-
-        ret = 0;
-        resolve_and_resume (frame, server_fsetattr_resume);
-out:
-        return ret;
-}
-
-
-int
-server_readlink (rpcsvc_request_t *req)
-{
-        server_state_t    *state                 = NULL;
-        call_frame_t      *frame                 = NULL;
-        gfs3_readlink_req  args                  = {{0,},};
-        int                ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_readlink_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_READLINK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        state->size  = args.size;
-
-        ret = 0;
-        resolve_and_resume (frame, server_readlink_resume);
-out:
-        return ret;
-}
-
-
-int
-server_create (rpcsvc_request_t *req)
-{
-        server_state_t  *state    = NULL;
-        call_frame_t    *frame    = NULL;
-        dict_t          *params   = NULL;
-        gfs3_create_req  args     = {{0,},};
-        int              ret      = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_create_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_CREATE;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        /* Unserialize the dictionary */
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, params,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->params = params;
-
-        state->resolve.bname  = gf_strdup (args.bname);
-        state->mode           = args.mode;
-        state->flags          = gf_flags_to_flags (args.flags);
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-
-        if (state->flags & O_EXCL) {
-                state->resolve.type = RESOLVE_NOT;
-        } else {
-                state->resolve.type = RESOLVE_DONTCARE;
-        }
-
-        ret = 0;
-        resolve_and_resume (frame, server_create_resume);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        if (params)
-                dict_unref (params);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-}
-
-
-int
-server_open (rpcsvc_request_t *req)
-{
-        server_state_t *state                 = NULL;
-        call_frame_t   *frame                 = NULL;
-        gfs3_open_req   args                  = {{0,},};
-        int             ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_open_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_OPEN;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        state->flags = gf_flags_to_flags (args.flags);
-
-        ret = 0;
-        resolve_and_resume (frame, server_open_resume);
-out:
-        return ret;
-}
-
-
-int
-server_readv (rpcsvc_request_t *req)
-{
-        server_state_t *state = NULL;
-        call_frame_t   *frame = NULL;
-        gfs3_read_req   args  = {{0,},};
-        int             ret   = -1;
-
-        if (!req)
-                goto out;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_read_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_READ;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = args.fd;
-        state->size           = args.size;
-        state->offset         = args.offset;
-        state->flags          = args.flag;
-
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_readv_resume);
-out:
-        return ret;
-}
-
-
-int
-server_writev (rpcsvc_request_t *req)
-{
-        server_state_t      *state  = NULL;
-        call_frame_t        *frame  = NULL;
-        gfs3_write_req       args   = {{0,},};
-        ssize_t              len    = 0;
-        int                  i      = 0;
-        int                  ret    = -1;
-
-        if (!req)
-                return ret;
-
-        len = xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_write_req);
-        if (len == 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_WRITE;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->offset        = args.offset;
-        state->flags         = args.flag;
-        state->iobref        = iobref_ref (req->iobref);
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        if (len < req->msg[0].iov_len) {
-                state->payload_vector[0].iov_base
-                        = (req->msg[0].iov_base + len);
-                state->payload_vector[0].iov_len
-                        = req->msg[0].iov_len - len;
-                state->payload_count = 1;
-        }
-
-        for (i = 1; i < req->count; i++) {
-                state->payload_vector[state->payload_count++]
-                        = req->msg[i];
-        }
-
-        for (i = 0; i < state->payload_count; i++) {
-                state->size += state->payload_vector[i].iov_len;
-        }
-
-        ret = 0;
-        resolve_and_resume (frame, server_writev_resume);
-out:
-        return ret;
-}
-
-
-int
-server_writev_vec (rpcsvc_request_t *req, struct iovec *payload,
-                   int payload_count, struct iobref *iobref)
-{
-        return server_writev (req);
-}
-
-#define SERVER3_1_VECWRITE_START 0
-#define SERVER3_1_VECWRITE_READINGHDR 1
-
-int
-server_writev_vecsizer (int state, ssize_t *readsize, char *addr)
-{
-        int nextstate = 0;
-        gfs3_write_req    write_req              = {{0,},};
-
-        switch (state) {
-        case SERVER3_1_VECWRITE_START:
-                *readsize = xdr_sizeof ((xdrproc_t) xdr_gfs3_write_req, &write_req);
-                nextstate = SERVER3_1_VECWRITE_READINGHDR;
-                break;
-        case SERVER3_1_VECWRITE_READINGHDR:
-                *readsize = 0;
-                nextstate = SERVER3_1_VECWRITE_START;
-                break;
-        default:
-                gf_log ("server3_1", GF_LOG_ERROR, "wrong state: %d", state);
-        }
-        return nextstate;
-}
-
-
-int
-server_release (rpcsvc_request_t *req)
-{
-        server_connection_t *conn = NULL;
-        gfs3_release_req     args = {{0,},};
-        gf_common_rsp        rsp  = {0,};
-        int                  ret  = -1;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_release_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        conn = req->trans->xl_private;
-        gf_fd_put (conn->fdtable, args.fd);
-
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-        ret = 0;
-out:
-        return ret;
-}
-
-int
-server_releasedir (rpcsvc_request_t *req)
-{
-        server_connection_t *conn = NULL;
-        gfs3_releasedir_req  args = {{0,},};
-        gf_common_rsp        rsp  = {0,};
-        int                  ret  = -1;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_release_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        conn = req->trans->xl_private;
-        gf_fd_put (conn->fdtable, args.fd);
-
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
-        ret = 0;
-out:
-        return ret;
-}
-
-
-int
-server_fsync (rpcsvc_request_t *req)
-{
-        server_state_t *state = NULL;
-        call_frame_t   *frame = NULL;
-        gfs3_fsync_req  args  = {{0,},};
-        int             ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fsync_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSYNC;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->flags         = args.data;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fsync_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_flush (rpcsvc_request_t *req)
-{
-        server_state_t *state = NULL;
-        call_frame_t   *frame = NULL;
-        gfs3_flush_req  args  = {{0,},};
-        int             ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_flush_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FLUSH;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_flush_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_ftruncate (rpcsvc_request_t *req)
-{
-        server_state_t     *state = NULL;
-        call_frame_t       *frame = NULL;
-        gfs3_ftruncate_req  args  = {{0,},};
-        int                 ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_ftruncate_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FTRUNCATE;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = args.fd;
-        state->offset         = args.offset;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_ftruncate_resume);
-out:
-        return ret;
-}
-
-
-int
-server_fstat (rpcsvc_request_t *req)
-{
-        server_state_t *state = NULL;
-        call_frame_t   *frame = NULL;
-        gfs3_write_req  args  = {{0,},};
-        int             ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fstat_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSTAT;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.fd_no   = args.fd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fstat_resume);
-out:
-        return ret;
-}
-
-
-int
-server_truncate (rpcsvc_request_t *req)
-{
-        server_state_t    *state                 = NULL;
-        call_frame_t      *frame                 = NULL;
-        gfs3_truncate_req  args                  = {{0,},};
-        int                ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_truncate_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_TRUNCATE;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-        state->offset        = args.offset;
-
-        ret = 0;
-        resolve_and_resume (frame, server_truncate_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_unlink (rpcsvc_request_t *req)
-{
-        server_state_t  *state                  = NULL;
-        call_frame_t    *frame                  = NULL;
-        gfs3_unlink_req  args                   = {{0,},};
-        int              ret                    = -1;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_unlink_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_UNLINK;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.bname  = gf_strdup (args.bname);
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_unlink_resume);
-out:
-        return ret;
-}
-
-
-int
-server_setxattr (rpcsvc_request_t *req)
-{
-        server_state_t      *state                 = NULL;
-        dict_t              *dict                  = NULL;
-        call_frame_t        *frame                 = NULL;
-        server_connection_t *conn                  = NULL;
-        gfs3_setxattr_req    args                  = {{0,},};
-        int32_t              ret                   = -1;
-        int                  op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_setxattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_SETXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type     = RESOLVE_MUST;
-        state->flags            = args.flags;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->dict = dict;
-
-        /* There can be some commands hidden in key, check and proceed */
-        gf_server_check_setxattr_cmd (frame, dict);
-
-        ret = 0;
-        resolve_and_resume (frame, server_setxattr_resume);
-
-        return ret;
-out:
-        if (dict)
-                dict_unref (dict);
-
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        return ret;
-}
-
-
-
-int
-server_fsetxattr (rpcsvc_request_t *req)
-{
-        server_state_t      *state                = NULL;
-        dict_t              *dict                 = NULL;
-        server_connection_t *conn                 = NULL;
-        call_frame_t        *frame                = NULL;
-        gfs3_fsetxattr_req   args                 = {{0,},};
-        int32_t              ret                  = -1;
-        int                  op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fsetxattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSETXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type      = RESOLVE_MUST;
-        state->resolve.fd_no     = args.fd;
-        state->flags             = args.flags;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->dict = dict;
-
-        ret = 0;
-        resolve_and_resume (frame, server_fsetxattr_resume);
-
-        return ret;
-out:
-        if (dict)
-                dict_unref (dict);
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-        return ret;
-}
-
-
-
-int
-server_fxattrop (rpcsvc_request_t *req)
-{
-        dict_t              *dict                 = NULL;
-        server_state_t      *state                = NULL;
-        server_connection_t *conn                 = NULL;
-        call_frame_t        *frame                = NULL;
-        gfs3_fxattrop_req    args                 = {{0,},};
-        int32_t              ret                  = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fxattrop_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FXATTROP;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->resolve.fd_no   = args.fd;
-        state->flags           = args.flags;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->dict = dict;
-
-        ret = 0;
-        resolve_and_resume (frame, server_fxattrop_resume);
-
-        return ret;
-
-out:
-        if (dict)
-                dict_unref (dict);
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        return ret;
-}
-
-
-
-int
-server_xattrop (rpcsvc_request_t *req)
-{
-        dict_t              *dict                  = NULL;
-        server_state_t      *state                 = NULL;
-        server_connection_t *conn                  = NULL;
-        call_frame_t        *frame                 = NULL;
-        gfs3_xattrop_req     args                  = {{0,},};
-        int32_t              ret                   = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        conn = req->trans->xl_private;
-
-        args.dict.dict_val = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_xattrop_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_XATTROP;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type    = RESOLVE_MUST;
-        state->flags           = args.flags;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->dict = dict;
-
-        ret = 0;
-        resolve_and_resume (frame, server_xattrop_resume);
-
-        return ret;
-out:
-        if (dict)
-                dict_unref (dict);
-
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        return ret;
-}
-
-
-int
-server_getxattr (rpcsvc_request_t *req)
-{
-        server_state_t      *state                 = NULL;
-        call_frame_t        *frame                 = NULL;
-        gfs3_getxattr_req    args                  = {{0,},};
-        int                  ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        args.name = alloca (256);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_getxattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_GETXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        if (args.namelen) {
-                state->name = gf_strdup (args.name);
-                /* There can be some commands hidden in key, check and proceed */
-                gf_server_check_getxattr_cmd (frame, state->name);
-        }
-
-        ret = 0;
-        resolve_and_resume (frame, server_getxattr_resume);
-out:
-        return ret;
-}
-
-
-int
-server_fgetxattr (rpcsvc_request_t *req)
-{
-        server_state_t      *state      = NULL;
-        call_frame_t        *frame      = NULL;
-        gfs3_fgetxattr_req   args       = {{0,},};
-        int                  ret        = -1;
-
-        if (!req)
-                return ret;
-
-        args.name = alloca (256);
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fgetxattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FGETXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type  = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        if (args.namelen)
-                state->name = gf_strdup (args.name);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fgetxattr_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_removexattr (rpcsvc_request_t *req)
-{
-        server_state_t       *state                 = NULL;
-        call_frame_t         *frame                 = NULL;
-        gfs3_removexattr_req  args                  = {{0,},};
-        int                   ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        args.name = alloca (256);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_removexattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_REMOVEXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-        state->name           = gf_strdup (args.name);
-
-        ret = 0;
-        resolve_and_resume (frame, server_removexattr_resume);
-out:
-        return ret;
-}
-
-int
-server_fremovexattr (rpcsvc_request_t *req)
-{
-        server_state_t       *state                 = NULL;
-        call_frame_t         *frame                 = NULL;
-        gfs3_fremovexattr_req  args                  = {{0,},};
-        int                   ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        args.name = alloca (4096);
-
-        if (!xdr_to_generic (req->msg[0], &args,
-                             (xdrproc_t)xdr_gfs3_fremovexattr_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FREMOVEXATTR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        state->resolve.fd_no  = args.fd;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-        state->name           = gf_strdup (args.name);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fremovexattr_resume);
-out:
-        return ret;
-}
-
-
-
-
-int
-server_opendir (rpcsvc_request_t *req)
-{
-        server_state_t   *state                 = NULL;
-        call_frame_t     *frame                 = NULL;
-        gfs3_opendir_req  args                  = {{0,},};
-        int               ret                   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_opendir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_OPENDIR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type   = RESOLVE_MUST;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_opendir_resume);
-out:
-        return ret;
-}
-
-
-int
-server_readdirp (rpcsvc_request_t *req)
-{
-        server_state_t      *state        = NULL;
-        call_frame_t        *frame        = NULL;
-        gfs3_readdirp_req    args         = {{0,},};
-        size_t               headers_size = 0;
-        int                  ret          = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_readdirp_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_READDIRP;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        /* FIXME: this should go away when variable sized iobufs are introduced
-         * and transport layer can send msgs bigger than current page-size.
-         */
-        headers_size = sizeof (struct rpc_msg) + sizeof (gfs3_readdir_rsp);
-        if ((frame->this->ctx->page_size < args.size)
-            || ((frame->this->ctx->page_size - args.size) < headers_size)) {
-                state->size = frame->this->ctx->page_size - headers_size;
-        } else {
-                state->size   = args.size;
-        }
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->offset = args.offset;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, state->dict,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-
-        ret = 0;
-        resolve_and_resume (frame, server_readdirp_resume);
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        return ret;
-}
-
-int
-server_readdir (rpcsvc_request_t *req)
-{
-        server_state_t      *state        = NULL;
-        call_frame_t        *frame        = NULL;
-        gfs3_readdir_req     args         = {{0,},};
-        size_t               headers_size = 0;
-        int                  ret          = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_readdir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_READDIR;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        /* FIXME: this should go away when variable sized iobufs are introduced
-         * and transport layer can send msgs bigger than current page-size.
-         */
-        headers_size = sizeof (struct rpc_msg) + sizeof (gfs3_readdir_rsp);
-        if ((frame->this->ctx->page_size < args.size)
-            || ((frame->this->ctx->page_size - args.size) < headers_size)) {
-                state->size = frame->this->ctx->page_size - headers_size;
-        } else {
-                state->size   = args.size;
-        }
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->offset = args.offset;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_readdir_resume);
-out:
-        return ret;
-}
-
-int
-server_fsyncdir (rpcsvc_request_t *req)
-{
-        server_state_t      *state = NULL;
-        call_frame_t        *frame = NULL;
-        gfs3_fsyncdir_req    args  = {{0,},};
-        int                  ret   = -1;
-
-        if (!req)
-                return ret;
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_fsyncdir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_FSYNCDIR;
-
-        state = CALL_STATE(frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        state->resolve.type = RESOLVE_MUST;
-        state->resolve.fd_no = args.fd;
-        state->flags = args.data;
-        memcpy (state->resolve.gfid, args.gfid, 16);
-
-        ret = 0;
-        resolve_and_resume (frame, server_fsyncdir_resume);
-out:
-        return ret;
-}
-
-
-
-int
-server_mknod (rpcsvc_request_t *req)
-{
-        server_state_t      *state                  = NULL;
-        call_frame_t        *frame                  = NULL;
-        dict_t              *params                 = NULL;
-        gfs3_mknod_req       args                   = {{0,},};
-        int                  ret                    = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_mknod_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_MKNOD;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, params,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->params = params;
-
-        state->resolve.type    = RESOLVE_NOT;
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-        state->resolve.bname   = gf_strdup (args.bname);
-
-        state->mode = args.mode;
-        state->dev  = args.dev;
-
-        ret = 0;
-        resolve_and_resume (frame, server_mknod_resume);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        if (params)
-                dict_unref (params);
-
-        /* memory allocated by libc, don't use GF_FREE */
-        if (args.dict.dict_val != NULL) {
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-
-}
-
-
-int
-server_mkdir (rpcsvc_request_t *req)
-{
-        server_state_t      *state                  = NULL;
-        call_frame_t        *frame                  = NULL;
-        dict_t              *params                 = NULL;
-        gfs3_mkdir_req       args                   = {{0,},};
-        int                  ret                    = -1;
-        int              op_errno = 0;
-
-        if (!req)
-                return ret;
-
-        args.bname = alloca (req->msg[0].iov_len);
-
-        if (!xdr_to_generic (req->msg[0], &args, (xdrproc_t)xdr_gfs3_mkdir_req)) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        frame = get_frame_from_request (req);
-        if (!frame) {
-                // something wrong, mostly insufficient memory
-                req->rpc_err = GARBAGE_ARGS; /* TODO */
-                goto out;
-        }
-        frame->root->op = GF_FOP_MKDIR;
-
-        state = CALL_STATE (frame);
-        if (!state->conn->bound_xl) {
-                /* auth failure, request on subvolume without setvolume */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-        GF_PROTOCOL_DICT_UNSERIALIZE (state->conn->bound_xl, params,
-                                      (args.dict.dict_val),
-                                      (args.dict.dict_len), ret,
-                                      op_errno, out);
-
-        state->params = params;
-
-        state->resolve.type    = RESOLVE_NOT;
-        memcpy (state->resolve.pargfid, args.pargfid, 16);
-        state->resolve.bname   = gf_strdup (args.bname);
-
-        state->mode = args.mode;
-
-        ret = 0;
-        resolve_and_resume (frame, server_mkdir_resume);
-
-        if (args.dict.dict_val != NULL) {
-                /* memory allocated by libc, don't use GF_FREE */
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-out:
-        if (op_errno)
-                req->rpc_err = GARBAGE_ARGS;
-
-        if (params)
-                dict_unref (params);
-
-        if (args.dict.dict_val != NULL) {
-                /* memory allocated by libc, don't use GF_FREE */
-                free (args.dict.dict_val);
-        }
-
-        return ret;
-}
-
-
-int
-server_rmdir (rpcsvc_request_t *req)
-{
-        server_state_t      *state                  = NULL;
-        call_frame_t        *frame                  = NULL;
-        gfs3_rmdir_req       args                   = {{0,},};
-        int                  ret                    = -1;
-
-        if (!req)
-
author	Manikandan Selvaganesh <mselvaga@redhat.com>	2015-10-05 12:59:18 +0530
committer	Dan Lambright <dlambrig@redhat.com>	2015-10-08 04:47:42 -0700
commit	37a553575ff8df7ede86e3c69fb48d77bfaccafe (patch)
tree	2adfe3304b58d76fe70a07aa64a1df6fc7d8d4c6 /cli/src/cli.h
parent	4cf1e84ec80fc58f6f978165a81fd8acbcc648f8 (diff)